def __init__(self): self.metadata = Metadata(True, 'lda') self.dataProvider = DataProvider() self.topicFile = 'tmp/topics.npy' self.kMeansFile = 'tmp/kmeans.pkl.npy'
log.debug("Input file: %s." % inputfile) log.debug("Original name: %s." % original) log.debug("IMDB ID: %s." % imdbid) info = converter.isValidSource(inputfile) if info: log.info("Processing %s." % inputfile) output = converter.process(inputfile, original=original, info=info) if output: # Tag with metadata try: tag = Metadata(MediaType.Movie, imdbid=imdbid, original=original, language=settings.taglanguage) if settings.tagfile: log.info('Tagging file with TMDB ID %s.', tag.tmdbid) tag.setHD(output['x'], output['y']) tag.writeTags(output['output'], settings.artwork, settings.thumbnail) except: log.exception("Unable to tag file") # QTFS if settings.relocate_moov: converter.QTFS(output['output']) # Copy to additional locations output_files = converter.replicate(output['output'])
# -*- coding: utf-8 -*- from common import logo, genres, repo_base_url from common import Track, get_files from metadata import Metadata print(logo) meta = Metadata() print("version " + meta.get_version() + "\n") def add_source(m3u_file, track): prettified_title = track.title.replace("-", " ") prettified_title = prettified_title.replace(".m3u", "") source = "#EXTINF:" + str(track.length) + "," + prettified_title + "\n" source = source + track.path + "\n" m3u_file += source return m3u_file for genre in genres: files = get_files(genre) m3u_file = "#EXTM3U\n" for file in files: track = Track(-1, file, (repo_base_url + "/" + genre + "/" + file)) m3u_file = add_source(m3u_file, track) file = open("../" + genre + "/" + "ALL_" + genre + ".m3u", "w") file.write(m3u_file)
def main(): parser = argparse.ArgumentParser( description='Maven based download index generator') parser.add_argument( '--webout', dest='output_web', default='/out', help= 'Base directory to output generated index pages. Will generate in sub-directories based on the maven path', type=parse_path) parser.add_argument( '--metaout', dest='output_meta', default='/out', help= 'Base directory to output generated metadata. Will generate in sub-directories based on the maven path', type=parse_path) parser.add_argument('--downloadroot', dest='dlroot', default='https://maven.minecraftforge.net/', help='Root URL for downloading artifacts') parser.add_argument('--webroot', dest='webroot', default='https://files.minecraftforge.net', help='Root URL for artifact pages') parser.add_argument( '--static', dest='static', default='https://files.minecraftforge.net/static/', help='Root URL for static assets used by the templates') parser.add_argument( '--folder', dest='folder', default='/in/repositories/releases/', help= 'Root directory for the maven structure to read metadata from files', type=parse_path) parser.add_argument('--config', dest='config', default='/in/global_overrides.json', help="Location of global_overrides.json file", type=parse_path) parser.add_argument('--templates', dest='templates', default='templates', type=parse_path, help="Path to templates") commands = parser.add_subparsers(help='Command to perform', dest='command', required=True) gen_command = commands.add_parser('gen', help='Indexes generator subcommand') gen_command.add_argument('artifact', help='Maven Artifact - net.minecraftforge:forge') index_command = commands.add_parser('index', help='Generate tracked project index') promote_command = commands.add_parser('promote', help='Promote subcommand') promote_command.add_argument( 'artifact', help='Maven Artifact - net.minecraftforge:forge') promote_command.add_argument('version', help='Maven Version') promote_command.add_argument('type', choices=['latest', 'recommended'], help='Type of promotion') args = parser.parse_args() print('Page Generator:') print(f'PyVer: {sys.version}') print(f'Folder: {args.folder}') print(f'Config: {args.config}') print(f'Web Out: {args.output_web}') print(f'Meta Out: {args.output_meta}') print(f'WebRoot: {args.webroot}') print(f'DLRoot: {args.dlroot}') print(f'Static: {args.static}') print(f'Templates:{args.templates}') print(f'Command: {args.command}') print(f'Artifact: {args.artifact if "artifact" in args else None}') print(f'Version: {args.version if "version" in args else None}') print(f'Type: {args.type if "type" in args else None}') metadata = Metadata(args.folder, args.output_meta, args.output_web, args.webroot, args.dlroot, args.static, args.config) artifact = Artifact.load_maven_xml( metadata, args.artifact) if 'artifact' in args else None templates = Templates(args.templates, args.static, args.webroot, args.dlroot) for gen in Generators[args.command]: gen.generate(metadata, artifact, templates, args)
from metadata import Metadata metadata = Metadata('/tmp/retile', 'label') def test__metadata(): _metadata = { 'name': 'name', 'label': 'label', 'provides_product_versions': [{ 'name': 'name' }], 'releases': [], 'property_blueprints': [], 'job_types': [] } metadata._mutate_metadata(_metadata) assert _metadata['name'] == 'name-label' assert _metadata['provides_product_versions'][0]['name'] == 'name-label' assert _metadata['label'] == 'label Label' def test__releases(): releases = [{ 'name': 'redis-enterprise', 'file': 'foo-bar-baz-3243151.23431.31143.pivotal' }, { 'name': 'not-redis-enterprise' }]
test_proportion = 0.25 print('input_size', input_size) print('record_size', reference_size) # Path setting data_path = '/vol/bitbucket/rh2515/MRI_Crohns' label_path = '/vol/bitbucket/rh2515/MRI_Crohns/labels' record_out_path = '/vol/bitbucket/rh2515/MRI_Crohns/tfrecords/ti_imb_generic' record_suffix = 'axial_t2_only' # Load data abnormal_cases = list(range(70)) healthy_cases = list(range(100)) metadata = Metadata(data_path, label_path, abnormal_cases, healthy_cases, dataset_tag='') # metadata = Metadata(data_path, label_path, abnormal_cases, healthy_cases, dataset_tag=' cropped') print('Loading images...') for patient in metadata.patients: print(f'Loading patient {patient.get_id()}') patient.load_image_data() # Preprocess data preprocessor = Preprocessor(constant_volume_size=reference_size) metadata.patients = preprocessor.process(metadata.patients, ileum_crop=False, region_grow_crop=True, statistical_region_crop=True)
def test_simple_tm(dut): """Testing the simple_tm module """ # start HW sim clock cocotb.fork(Clock(dut.axis_aclk, PERIOD).start()) # Reset the DUT dut._log.debug("Resetting DUT") dut.axis_resetn <= 0 yield ClockCycles(dut.axis_aclk, 10) dut.axis_resetn <= 1 dut._log.debug("Out of reset") # wait for the pifo to finish resetting yield wait_pifo_busy(dut) yield ClockCycles(dut.axis_aclk, 100) dut.m_axis_tready <= 0 # build the list of pkts and metadata to insert pkts_meta_in = [] for i in range(NUM_PKTS): # pkt_len = random.randint(50, 1000) # build a packet pkt = Ether(dst='aa:aa:aa:aa:aa:aa', src='bb:bb:bb:bb:bb:bb') pkt = pkt / ('\x11' * 18 + '\x22' * 32) rank = random.randint(0, 100) # build the metadata meta = Metadata(pkt_len=len(pkt), src_port=0b00000001, dst_port=0b00000100, rank=rank) tuser = BinaryValue(bits=len(meta) * 8, bigEndian=False) tuser.set_buff(str(meta)) pkts_meta_in.append((rank, pkt, tuser)) ranks_in = [tup[0] for tup in pkts_meta_in] pkts_in = [tup[1] for tup in pkts_meta_in] meta_in = [tup[2] for tup in pkts_meta_in] # Attach an AXI4Stream Master to the input pkt interface pkt_master = AXI4StreamMaster(dut, 's_axis', dut.axis_aclk) pkt_in_stats = AXI4StreamStats(dut, 's_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) pkt_in_stats_thread = cocotb.fork( pkt_in_stats.record_n_delays(len(pkts_in))) # Send pkts and metadata in the HW sim yield pkt_master.write_pkts(pkts_in, meta_in) # wait for pifo to no longer be busy yield wait_pifo_busy(dut) # delay between writing pkts and reading them out yield ClockCycles(dut.axis_aclk, 25) # wait for the pifo to finish the final enqueue yield FallingEdge(dut.axis_aclk) while dut.simple_tm_inst.pifo_busy.value: yield RisingEdge(dut.axis_aclk) yield FallingEdge(dut.axis_aclk) yield RisingEdge(dut.axis_aclk) # Attach an AXI4StreamSlave to the output pkt interface pkt_slave = AXI4StreamSlave(dut, 'm_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) pkt_out_stats = AXI4StreamStats(dut, 'm_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) pkt_out_stats_thread = cocotb.fork( pkt_out_stats.record_n_delays(len(pkts_in))) # Read pkts out yield pkt_slave.read_n_pkts(len(pkts_in)) # # wait for stats threads to finish # yield pkt_in_stats_thread.join() # yield pkt_out_stats_thread.join() sorted_pkts_meta = sorted(pkts_meta_in, key=lambda x: x[0]) expected_ranks = [tup[0] for tup in sorted_pkts_meta] expected_pkts = [tup[1] for tup in sorted_pkts_meta] expected_meta = [tup[2] for tup in sorted_pkts_meta] pkts_out = pkt_slave.pkts meta_out = pkt_slave.metadata actual_ranks = [Metadata(m.get_buff()).rank for m in meta_out] print 'input ranks = {}'.format(ranks_in) print 'expected output ranks = {}'.format(expected_ranks) print 'actual output ranks = {}'.format(actual_ranks) print '' print 'pkt_in_delays = {}'.format(pkt_in_stats.delays) print 'pkt_out_delays = {}'.format(pkt_out_stats.delays) print '\tmax = {}'.format(max(pkt_out_stats.delays)) print '\tavg = {}'.format( sum(pkt_out_stats.delays) / float(len(pkt_out_stats.delays))) print '\tmin = {}'.format(min(pkt_out_stats.delays)) results = {} results['enq_delays'] = pkt_in_stats.delays results['deq_delays'] = pkt_out_stats.delays with open(RESULTS_FILE, 'w') as f: json.dump(results, f) error = False for (exp_pkt, pkt, exp_meta, meta, i) in zip(expected_pkts, pkts_out, expected_meta, meta_out, range(len(expected_pkts))): if str(exp_pkt) != str(pkt): print 'ERROR: exp_pkt != pkt_out for pkt {}'.format(i) error = True if exp_meta.get_buff() != meta.get_buff(): print 'ERROR: exp_meta != meta_out for pkt {}'.format(i) exp_meta = Metadata(exp_meta.get_buff()) meta = Metadata(meta.get_buff()) print 'exp_meta = {}'.format(exp_meta.summary()) print 'meta = {}'.format(meta.summary()) error = True print "******************" print "Checking for duplicates:" print "******************" for r, i in zip(actual_ranks, range(len(actual_ranks))): try: ranks_in.remove(r) except ValueError as e: print 'ERROR: output rank ({}) not in input set'.format(r) print e error = True if len(ranks_in) > 0: print 'ERROR: not all ranks removed: {}'.format(ranks_in) error = True yield ClockCycles(dut.axis_aclk, 20) if error: print 'ERROR: Test Failed' raise (TestFailure)
def onPlayBackStarted(self): # pylint: disable=invalid-name """Called when user starts playing a file""" self.path = getInfoLabel(self.path_infolabel) if self.path.startswith('plugin://plugin.video.vrt.nu/'): self.listen = True else: self.listen = False return log(3, '[PlayerInfo {id}] Event onPlayBackStarted', id=self.thread_id) # Set property to let wait_for_resumepoints function know that update resume is busy set_property('vrtnu_resumepoints', 'busy') # Update previous episode when using "Up Next" if self.path.startswith('plugin://plugin.video.vrt.nu/play/upnext'): self.push_position(position=self.last_pos, total=self.total) # Reset episode data self.asset_id = None self.title = None self.url = None self.whatson_id = None ep_id = play_url_to_id(self.path) # Avoid setting resumepoints for livestreams for channel in CHANNELS: if ep_id.get('video_id') and ep_id.get('video_id') == channel.get( 'live_stream_id'): log(3, '[PlayerInfo {id}] Avoid setting resumepoints for livestream {video_id}', id=self.thread_id, video_id=ep_id.get('video_id')) self.listen = False # Reset vrtnu_resumepoints property before return set_property('vrtnu_resumepoints', None) return # Get episode data needed to update resumepoints from VRT NU Search API episode = self.apihelper.get_single_episode_data( video_id=ep_id.get('video_id'), whatson_id=ep_id.get('whatson_id'), video_url=ep_id.get('video_url')) # Avoid setting resumepoints without episode data if episode is None: # Reset vrtnu_resumepoints property before return set_property('vrtnu_resumepoints', None) return from metadata import Metadata self.asset_id = Metadata(None, None).get_asset_id(episode) self.title = episode.get('program') self.url = url_to_episode(episode.get('url', '')) self.whatson_id = episode.get( 'whatsonId') or None # Avoid empty string # Kodi 17 doesn't have onAVStarted if kodi_version_major() < 18: self.onAVStarted()
def get_mismatches(self, reference_metadata, target_metadata, traverse_sub_workflows=False): """ Takes two metadata files (both belonging to a common workflow execution), iterates through the outputs of their task, downloads the objects if not already exist in the working directory, compares the corresponding files, and returns the files that do not match. """ def record_compare_result(match, reference, target): if not match: if call not in mismatches: mismatches[call] = [] mismatches[call].append([reference, target]) # First we define a method that takes a list # of a task outputs, and keeps only those that # are files and their extension match the # file types that we want to compare # (e.g., filter only VCF files). filter_method = FilterBasedOnExtensions( self.filetypes_to_compare.keys()).filter # Then we create two instances of the Metadata # class, one for each metadata file, and we # invoke the `get_outputs` method which traverses # the outputs of task, and returns those filtered # by the above-defined filter. ref_output_files = Metadata(reference_metadata).get_outputs( traverse_sub_workflows, filter_method) test_output_files = Metadata(target_metadata).get_outputs( traverse_sub_workflows, filter_method) mismatches = {} i = 0 r_t = ref_output_files.keys() - test_output_files.keys() t_r = test_output_files.keys() - ref_output_files.keys() if r_t or t_r: print(f"\n{COLOR_BLINKING}WARNING!{COLOR_ENDC}") print(f"The reference and test metadata files differ " f"in their outputs; " f"{COLOR_ULINE}the differences will be skipped.{COLOR_ENDC}") if r_t: print(f"\t{len(r_t)}/{len(ref_output_files.keys())} " f"outputs of the reference are not in the test:") for x in r_t: print(f"\t\t- {x}") if t_r: print(f"\t{len(t_r)}/{len(test_output_files.keys())} " f"outputs of the test are not in the reference:") for x in t_r: print(f"\t\t- {x}") print("\n") [ref_output_files.pop(x) for x in r_t] print(f"{COLOR_YELLOW}Comparing {len(ref_output_files)} " f"files that are common between reference and test " f"metadata files and their respective task is executed " f"successfully.{COLOR_ENDC}") for call, ref_outputs in ref_output_files.items(): i += 1 matched = True print(f"Comparing\t{i}/{len(ref_output_files)}\t{call} ... ", end="") for extension, objs in ref_outputs.items(): if len(objs) != len(test_output_files[call][extension]): record_compare_result(False, objs, test_output_files[call][extension]) matched = False continue for idx, obj in enumerate(objs): equals, x, y = \ self.filetypes_to_compare[extension].equals( obj, test_output_files[call][extension][idx]) record_compare_result(equals, x, y) if not equals: matched = False if matched: print(f"{COLOR_GREEN}match{COLOR_ENDC}") else: print(f"{COLOR_RED}mismatch{COLOR_ENDC}") return mismatches
def load_fnames(md_path, zindexes): md = Metadata(md_path) md.image_table = md.image_table[[ True if (('hybe' in i) or ('nucstain' in i)) else False for i in md.image_table.acq ]] posnames = md.posnames # Speed up checking if a position is already fully done not_finished = [] finished_pos = [] for pos in posnames: try: processing = pickle.load( open( os.path.join(md_path, 'codestacks', pos, 'processing.pkl'), 'rb')) if len(processing ) == 18: # Check to make sure position is finished finished_pos.append(pos) else: not_finished.append(pos) except: not_finished.append(pos) random_posnames = np.random.choice(not_finished, size=len(not_finished), replace=False) hybe_list = sorted([ i.split('_')[0] for i in md.acqnames if ('hybe' in i) or ('nucstain' in i) ]) if zindexes == -1: Input = [{ 'fname_dicts': md.stkread(Channel='DeepBlue', Position=pos, fnames_only=True, groupby='acq', hybe=hybe_list), 'posnames': pos } for pos in random_posnames] else: Input = [{ 'fname_dicts': md.stkread(Channel='DeepBlue', Position=pos, fnames_only=True, groupby='acq', hybe=hybe_list, Zindex=list(range(8, 25))), 'posnames': pos } for pos in random_posnames] # Input = [] # for pos in random_posnames: # pos_df = {} # for hybe in hybe_list: # pos_df[hybe] = [] # for z in range(1,zindexes): # fnames = md.stkread(Channel='DeepBlue', Position=pos, # fnames_only=True, groupby='acq', # hybe=hybe_list,Zindex=z) # for h,f in fnames.items(): # pos_df[h.split('_')[0]].append(f) # Input.append({'fname_dicts':pos_df,'posnames': pos}) return Input
def get_metadata(file_path): return Metadata(file_path)
def test_pifo_pkt_storage(dut): """Testing pifo_pkt_storage module """ # instantiate the interface to the simpy implemenation env = simpy.Environment() ps_simpy = PS_simpy_iface(env) # start HW sim clock cocotb.fork(Clock(dut.axis_aclk, PERIOD).start()) # Reset the DUT dut._log.debug("Resetting DUT") dut.axis_resetn <= 0 dut.m_axis_pkt_tready <= 0 yield ClockCycles(dut.axis_aclk, 10) dut.axis_resetn <= 1 dut.m_axis_pkt_tready <= 1 dut._log.debug("Out of reset") # initialize the read request pointers dut.s_axis_ptr_tvalid <= 0 dut.s_axis_ptr_tdata <= 0 dut.s_axis_ptr_tlast <= 0 # Attach an AXI4Stream Master to the input pkt interface pkt_master = AXI4StreamMaster(dut, 's_axis_pkt', dut.axis_aclk) # Attach and AXI4StreamSlave to the output ptr interface ptr_slave = AXI4StreamSlave(dut, 'm_axis_ptr', dut.axis_aclk) hw_results = {} hw_results['out_ptrs'] = [] # build the list of pkts and metadata to insert all_pkts = [] all_meta = [] for i in range(10): pkt_len = random.randint(50, 1000) # build a packet pkt = Ether(dst='aa:aa:aa:aa:aa:aa', src='bb:bb:bb:bb:bb:bb') # pkt = pkt / ('\x11'*18 + '\x22'*32 + '\x33'*32 + '\x44'*32 + '\x55'*16) pkt = pkt / ('\x11'*(pkt_len - 14)) # build the metadata meta = Metadata(pkt_len=len(pkt), src_port=0b00000001, dst_port=0b00000100) all_pkts.append(pkt) all_meta.append(meta) # send pkts / metadata and read resulting pointers for p, m in zip(all_pkts, all_meta): # Start reading for output ptrs ptr_slave_thread = cocotb.fork(ptr_slave.read()) # send the pkts in the HW sim tuser = BinaryValue(bits=len(m)*8, bigEndian=False) tuser.set_buff(str(m)) yield pkt_master.write_pkts([p], [tuser]) # wait to finish reading pointers yield ptr_slave_thread.join() # ptr_slave.data is all of the ptr words that have been read so far hw_results['out_ptrs'] = ptr_slave.data # check results with simpy simulation env.process(ps_simpy.insert_pkts(all_pkts, all_meta, hw_results)) env.run() if ps_simpy.test_failed: raise TestFailure('Test Failed') # pause between pkt insertions and removals yield ClockCycles(dut.axis_aclk, 10) # Attach an AXI Stream master to read request interface ptr_master = AXI4StreamMaster(dut, 's_axis_ptr', dut.axis_aclk) # Attach an AXI Stream slave to the output packet interface pkt_slave = AXI4StreamSlave(dut, 'm_axis_pkt', dut.axis_aclk) # remove the inserted pkts hw_results['out_pkts'] = [] for ptrs in hw_results['out_ptrs']: # start reading for output pkts pkt_slave_thread = cocotb.fork(pkt_slave.read_pkt()) # submit read request yield ptr_master.write([ptrs]) # wait to finish reading pkt yield pkt_slave_thread.join() yield RisingEdge(dut.axis_aclk) hw_results['out_pkts'] = pkt_slave.pkts hw_results['out_meta'] = [Metadata(m.get_buff()) for m in pkt_slave.metadata] # verify that the removed pkts are the same as the ones that were inserted if len(all_pkts) != len(hw_results['out_pkts']): print 'ERROR: {} pkts inserted, {} pkts removed'.format(len(all_pkts), len(hw_results['out_pkts'])) raise TestFailure('Test Failed') for (pkt_in, pkt_out, meta_in, meta_out, i) in zip(all_pkts, hw_results['out_pkts'], all_meta, hw_results['out_meta'], range(len(all_pkts))): if str(pkt_in) != str(pkt_out): print 'ERROR: pkt_in != pkt_out for pkt {}'.format(i) print 'len(pkt_in) = {}, pkt_in: {}'.format(len(pkt_in), pkt_in.summary()) print 'len(pkt_out) = {}, pkt_out: {}'.format(len(pkt_out), pkt_out.summary()) # raise TestFailure('Test Failed') if str(meta_in) != str(meta_out): print 'ERROR: meta_in != meta_out for pkt {}'.format(i) print '\tmeta_in = {}'.format(meta_in.summary()) print '\tmeta_out = {}'.format(meta_out.summary()) # raise TestFailure('Test Failed') yield ClockCycles(dut.axis_aclk, 20)
def __init__(self): self.metadata = Metadata()
def test_both_enqdeq(dut): """Testing the simple_tm module with a constant fill level """ # start HW sim clock cocotb.fork(Clock(dut.axis_aclk, PERIOD).start()) # Reset the DUT dut._log.debug("Resetting DUT") dut.axis_resetn <= 0 yield ClockCycles(dut.axis_aclk, 10) dut.axis_resetn <= 1 dut._log.debug("Out of reset") # wait for the pifo to finish resetting yield FallingEdge(dut.axis_aclk) while dut.simple_tm_inst.pifo_busy.value: yield RisingEdge(dut.axis_aclk) yield FallingEdge(dut.axis_aclk) yield RisingEdge(dut.axis_aclk) yield ClockCycles(dut.axis_aclk, 100) dut.m_axis_tready <= 0 # build the list of pkts and metadata to insert pkts_meta_in = make_pkts_and_meta(FILL_LEVEL - 1) ranks_in = [tup[0] for tup in pkts_meta_in] pkts_in = [tup[1] for tup in pkts_meta_in] meta_in = [tup[2] for tup in pkts_meta_in] # Attach an AXI4Stream Master to the input pkt interface pkt_master = AXI4StreamMaster(dut, 's_axis', dut.axis_aclk) # Send pkts and metadata in the HW sim yield pkt_master.write_pkts(pkts_in, meta_in) # wait a few cycles before begining measurements yield ClockCycles(dut.axis_aclk, 25) # Attach and AXI4StreamSlave to the output pkt interface pkt_slave = AXI4StreamSlave(dut, 'm_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) # connect stats tools pkt_in_stats = AXI4StreamStats(dut, 's_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) pkt_out_stats = AXI4StreamStats(dut, 'm_axis', dut.axis_aclk, idle_timeout=IDLE_TIMEOUT) expected_outputs = [] enq_delays = [] deq_delays = [] for i in range(NUM_SAMPLES): data = make_pkts_and_meta(1) pkts_in = [tup[1] for tup in data] meta_in = [tup[2] for tup in data] # compute expected outputs expected_outputs.append(min(pkts_meta_in)) pkts_meta_in.remove(min(pkts_meta_in)) # The removal happens after the insertion pkts_meta_in += data # # start recording stats # pkt_in_stats_thread = cocotb.fork(pkt_in_stats.record_n_delays(1)) # pkt_out_stats_thread = cocotb.fork(pkt_out_stats.record_n_delays(1)) # send in packet pkt_master_thread = cocotb.fork(pkt_master.write_pkts( pkts_in, meta_in)) # Read out packet pkt_slave_thread = cocotb.fork(pkt_slave.read_n_pkts(1)) yield pkt_master_thread.join() yield pkt_slave_thread.join() # # record results # enq_delays += pkt_in_stats.delays # deq_delays += pkt_out_stats.delays # wait a few cycles between samples yield ClockCycles(dut.axis_aclk, 30) if pkt_slave.error: print "ERROR: pkt_slave timed out" break sorted_pkts_meta = sorted(pkts_meta_in, key=lambda x: x[0]) expected_ranks = [tup[0] for tup in expected_outputs] expected_pkts = [tup[1] for tup in expected_outputs] expected_meta = [tup[2] for tup in expected_outputs] pkts_out = pkt_slave.pkts meta_out = pkt_slave.metadata actual_ranks = [Metadata(m.get_buff()).rank for m in meta_out] print 'input_ranks = {}'.format(input_ranks) print 'expected output ranks = {}'.format(expected_ranks) print 'actual output ranks = {}'.format(actual_ranks) print '' print 'pkt_in_delays = {}'.format(enq_delays) print 'pkt_out_delays = {}'.format(deq_delays) results = {} results['enq_delays'] = enq_delays results['deq_delays'] = deq_delays with open(RESULTS_FILE, 'w') as f: json.dump(results, f) error = False for (exp_pkt, pkt, exp_meta, meta, i) in zip(expected_pkts, pkts_out, expected_meta, meta_out, range(len(expected_pkts))): if str(exp_pkt) != str(pkt): print 'ERROR: exp_pkt != pkt_out for pkt {}'.format(i) error = True if exp_meta.get_buff() != meta.get_buff(): print 'ERROR: exp_meta != meta_out for pkt {}'.format(i) exp_meta = Metadata(exp_meta.get_buff()) meta = Metadata(meta.get_buff()) print 'exp_meta = {}'.format(exp_meta.summary()) print 'meta = {}'.format(meta.summary()) error = True yield ClockCycles(dut.axis_aclk, 20) if error: print 'ERROR: Test Failed' raise (TestFailure)
def __init__(self): """ Attributes ---------- metadata: dict presentation metadata; each element of the dictionary if a dict with ['value', 'user'] items: value contains the metadata value and user indicates if the value comes from user (if True) or from defaults (if False). """ self.reset() self.metadata = {'title': Metadata(name='title', value=''), 'subtitle': Metadata(name='subtitle', value=''), 'authors': Metadata(name='authors', value=[]), 'authors_short': Metadata(name='authors_short', value=[]), 'emails': Metadata(name='emails', value=[]), 'affiliations': Metadata(name='affiliations', value=[]), 'affiliations_short': Metadata(name='affiliations_short', value=[]), 'logo': Metadata(name='logo', value=''), 'timer': Metadata(name='timer', value=''), 'location': Metadata(name='location', value=''), 'location_short': Metadata(name='location_short', value=''), 'date': Metadata(name='date', value=''), 'conference': Metadata(name='conference', value=''), 'conference_short': Metadata(name='conference_short', value=''), 'session': Metadata(name='session', value=''), 'session_short': Metadata(name='session_short', value=''), 'max_time': Metadata(name='max_time', value='25'), 'total_slides_number': Metadata(name='total_slides_number', value=''), 'dirs_to_copy': Metadata(name='dirs_to_copy', value=[]), 'toc': Metadata(name='toc', value=OrderedDict()), 'toc_depth': Metadata(name='toc_depth', value='2'), 'chaptertitle': Metadata(name='chaptertitle', value=''), 'chapternumber': Metadata(name='chapternumber', value=''), 'sectiontitle': Metadata(name='sectiontitle', value=''), 'sectionnumber': Metadata(name='sectionnumber', value=''), 'subsectiontitle': Metadata(name='subsectiontitle', value=''), 'subsectionnumber': Metadata(name='subsectionnumber', value=''), 'slidetitle': Metadata(name='slidetitle', value=''), 'slidenumber': Metadata(name='slidenumber', value=''), 'css_overtheme': Metadata(name='css_overtheme', value=[]), 'custom': Metadata(name='custom-[0-9]*', value='')} self.theme = Theme() self.parser = Parser() self.chapters = [] self.position = Position() return
from file_manager import FileManager from worker import Worker from notifier import Notifier def root_configurer(log_queue, level): h = handlers.QueueHandler(log_queue) root = logging.getLogger() root.addHandler(h) root.setLevel(level) if __name__ == "__main__": processes = list() config = Config() metadata = Metadata(config.metadata_file) log_queue = multiprocessing.Queue(-1) listener = Listener(log_queue) # listener.start() root_configurer(log_queue, config.logger_level) logger = logging.getLogger(__name__) pending_tasks = multiprocessing.Queue(-1) done_tasks = multiprocessing.Queue(-1) error_tasks = multiprocessing.Queue(-1) slack_queue = multiprocessing.Queue(-1) processes.append( FileManager(log_queue, pending_tasks, done_tasks, error_tasks, slack_queue)) processes.append(Notifier(log_queue, slack_queue))
def run_modules(self): ag = int(self.gv.settings.args['--aggression']) self.debug.print_debug( self, u'Running at aggression level {0} {1}'.format( ag, "[grrr!]" if ag == 10 else "")) if ag > 10: self.debug.print_debug( self, "WARNING: safety bail-out features are disabled at aggression level 11" ) if self.args['bibscan']: BibliographyDatabase(self.gv).scan() else: # check for stylesheets self.gv.check_file_exists(self.gv.docx_style_sheet_dir) # metadata file gv.metadata_file = self.set_metadata_file() self.gv.mk_dir(self.gv.output_folder_path) if self.args['doc']: # run doc to docx conversion # then run docx to tei UnoconvToDocx(self.gv).run('doc') DocxToTei(self.gv).run(True, self.args['--proprietary']) elif self.args['odt']: # run odt to docx conversion # then run docx to tei UnoconvToDocx(self.gv).run('odt') DocxToTei(self.gv).run(True, self.args['--proprietary']) elif self.args['other']: # run other unoconv-supported format to docx conversion # then run docx to tei UnoconvToDocx(self.gv).run('unoconv') DocxToTei(self.gv).run(True, self.args['--proprietary']) elif self.args['docx']: # run docx to tei conversion # includes hooks for proprietary transforms if enabled DocxToTei(self.gv).run(True, self.args['--proprietary']) elif self.args['docxextracted']: self.debug.print_debug(self, u'Skipping docx extraction') DocxToTei(self.gv).run(False, self.args['--proprietary']) elif self.args['tei']: self.debug.print_debug( self, u'Skipping docx extraction; processing TEI file') DocxToTei(self.gv).run(False, self.args['--proprietary'], tei=True) if self.args['--puretei']: self.debug.print_debug(self, u'Exiting as TEI transform complete') return metadata = Metadata(self.gv) metadata.pre_clean() # run size classifier # aggression 5 SizeClassifier(self.gv).run() # run bibliographic addins handler # aggression 4 found_bibliography = BibliographyAddins(self.gv).run() # run list classifier # aggression 4 ListClassifier(self.gv).run() bibliography_classifier = BibliographyClassifier(self.gv) if not found_bibliography: # run bibliographic classifier # aggression 4 bibliography_classifier.run() # tei # aggression 3 TeiManipulate(self.gv).run() # run tei to nlm conversion TeiToNlm(self.gv).run(not found_bibliography) if self.gv.settings.args['--purenlm']: self.debug.print_debug(self, u'Exiting as NLM transform complete') return manipulate = NlmManipulate(self.gv) if not self.gv.used_list_method: manipulate.fuse_references() # run reference linker if not (self.args['--nolink']): rl = ReferenceLinker(self.gv) rl.run(self.args['--interactive']) rl.cleanup() # run table classifier cc = CaptionClassifier(self.gv) if int(self.args['--aggression']) > int( self.gv.settings.get_setting( 'tablecaptions', self, domain='aggression')): cc.run_tables() if int(self.args['--aggression']) > int( self.gv.settings.get_setting( 'graphiccaptions', self, domain='aggression')): cc.run_graphics() # run metadata merge metadata.run() if self.args['--interactive']: bibliography_classifier.run_prompt(True) # process any bibliography entries that are possible BibliographyDatabase(self.gv).run() # remove stranded titles and cleanup manipulate.final_clean() if self.args['--identifiers']: IdGenerator(self.gv).run() if self.args['--chain']: # construct and run an XSLT chainer XslChain(self.gv).run() if self.args['--clean']: ComplianceEnforcer(self.gv).run()
def __init__(self, _favorites, _resumepoints): """Constructor for the ApiHelper class""" self._favorites = _favorites self._resumepoints = _resumepoints self._metadata = Metadata(_favorites, _resumepoints)
if args.md_path == 'None': deconvolved_path = os.path.join(args.analysis_path, 'deconvolved') if not os.path.exists(deconvolved_path): print('analysis path doesnt have your metadata') print('add path to metadata after analysis path') else: deconvolved_path = args.md_path bead_path = os.path.join(args.analysis_path, 'beads') if not os.path.exists(bead_path): os.makedirs(bead_path) results_path = os.path.join(args.analysis_path, 'results') if not os.path.exists(results_path): os.makedirs(results_path) #Retreiving file names and position names md = Metadata(deconvolved_path) posnames = md.image_table.Position.unique() posnames = [pos for pos in posnames if "Pos" in pos] hybe_list = [ 'hybe1', 'hybe2', 'hybe3', 'hybe4', 'hybe5', 'hybe6', 'hybe7', 'hybe8', 'hybe9' ] fnames_dicts = [ md.stkread(Channel='DeepBlue', Position=pos, fnames_only=True, groupby='acq', hybe=hybe_list) for pos in posnames ] #should move this to a better place probably just replace pyspots ave bead and import it Ave_Bead = pickle.load(
if __name__ == '__main__': os.environ['MKL_NUM_THREADS'] = '4' os.environ['GOTO_NUM_THREADS'] = '4' os.environ['OMP_NUM_THREADS'] = '4' print(args) # Assuming these get imported during call below: # 1. bitmap # 2. bids, blanks, gids, cwords, gene_codeword_vectors, blank_codeword_vectors # 3. norm_gene_codeword_vectors, norm_blank_codeword_vectors cstk_path = args.cstk_path ncpu = args.ncpu niter = args.niter mask_type = args.mask_type new_mask = args.new_mask md = Metadata(args.md_path) seqfish_config = importlib.import_module(args.cword_config) bitmap = seqfish_config.bitmap normalized_gene_vectors = seqfish_config.norm_gene_codeword_vectors hybedatas = [(i, HybeData(os.path.join(cstk_path, i))) for i in os.listdir(cstk_path) if os.path.isdir(os.path.join(cstk_path, i))] # Note preceding blocks and this can be noisy if restarted after crash etccc # Note preceding blocks and this can be noisy if restarted after crash etccc with multiprocessing.Pool(ncpu) as ppool: failed_positions = [] for i in range(niter): print('N Positions left: ', len(hybedatas))
def __init__(self): ''' Initializes TV-guide object ''' self._favorites = Favorites() self._resumepoints = ResumePoints() self._metadata = Metadata(self._favorites, self._resumepoints) install_opener(build_opener(ProxyHandler(get_proxies())))
def main(): args = parse_args() init_log(args.log) with open(args.app) as app_file: # parse and validate the requested data application JSON file application = Application(app_file) logging.info("Input data application parsed: {}".format(args.app)) # Create output directory for the results application_dir = create_app_dir(application) # check what data types are allowed for this application allowed_data_types = application.allowed_data_types() logging.info("Allowed data types: {}".format( ' '.join(allowed_data_types))) if len(allowed_data_types) > 0: # Get all the sample metadata for all requested cohorts requested_cohorts = application.cohorts() metadata = Metadata(args.data, requested_cohorts) logging.info("Metadata collected for requested cohorts: {}".format( ' '.join(requested_cohorts))) metadata_sample_ids = sorted(metadata.get_sample_ids()) logging.info("Metadata for sample IDs: {}".format( ' '.join(metadata_sample_ids))) # Filter the sample metadata based on patient consent metadata.filter_consent(args.consent, allowed_data_types) logging.warning("Consent not handled yet. FIXME") # Find all the file paths for requested file types for each # consented sample requested_file_types = application.file_types() logging.info("Requested file types: {}".format( ' '.join(requested_file_types))) fastqs, bams, bais, vcfs = get_files(args.data, requested_file_types, metadata) logging.info("VCF files selected:\n{}".format('\n'.join(vcfs))) logging.info("BAM files selected:\n{}".format('\n'.join(bams))) logging.info("BAI files selected:\n{}".format('\n'.join(bais))) logging.info("FASTQ files selected:\n{}".format('\n'.join(fastqs))) output_files = [] if 'Anonymised' in allowed_data_types: # generate random IDs for all output samples randomised_ids = make_random_ids(args.usedids, metadata.sample_ids) metadata.anonymise(randomised_ids) metadata.write(args.metaout) logging.info("Anonymised metadata written to: {}".format( args.metaout)) new_vcfs = anonymise_files(vcfs, randomised_ids, application_dir, VCF_filename, vcf_edit) new_bams = anonymise_files(bams, randomised_ids, application_dir, BAM_filename, bam_edit) # BAIs and FASTQs are just sym-linked to output with randomised name new_bais = anonymise_files(bais, randomised_ids, application_dir, BAI_filename) new_fastqs = anonymise_files(fastqs, randomised_ids, application_dir, FASTQ_filename) output_files.extend(new_vcfs + new_bams + new_bais + new_fastqs) logging.info("Output files are anonymised") elif 'Re-identifiable' in allowed_data_types: new_links = link_files(application_dir, vcfs + bams + bais + fastqs) output_files.extend(new_links) logging.info( "Files linked in directory: {}".format(application_dir)) metadata.write(args.metaout) logging.info("Output files are re-identifiable") else: print_error( "Allowed data is neither anonymised nor re-identifiable") exit(ERROR_BAD_ALLOWED_DATA) logging.info("Generating MD5 checksums on output files") md5_files(args.md5, output_files) else: logging.warning("No data available for this application")
def FromChat(chat, min_period, max_period, logger): meta = Metadata(chat.id, chat.type, get_chat_title(chat)) vocab = Generator() return Reader(meta, vocab, min_period, max_period, logger)
def test_hash_length(self): """ Tests the length of the md5 hash """ meta = Metadata(0, 0, 0, DIGEST) self.assertTrue(len(DIGEST) == len(meta.hash))
print('Starting...', 'already done ', str(len(doneso)), ', but ', str(len(all_images)), 'left') with open(os.open(fn, os.O_CREAT | os.O_WRONLY, 0o775), 'a') as f: for result in p.imap(pfunc, all_images, chunksize=chunksize): f.write(str(result)+'\n') tend = time.time() print('Finished processing ', str(len(all_images)), 'images in ', str(tend-tstart), 'seconds.') if __name__ == "__main__": ncpu = args.ncpu md_path = args.md_path out_path = args.out_path if isinstance(ncpu, list): assert(len(ncpu)==1) ncpu = ncpu[0] md = Metadata(md_path) print('Number of images total: ', len(md.image_table.filename)) base_path = md.base_pth if not base_path[-1]=='/': base_path=base_path+'/' #print(base_path) fn = os.path.join(out_path, 'processing.log') chunksize=100 os.environ['MKL_NUM_THREADS'] = '1' os.environ['GOTO_NUM_THREADS'] = '1' os.environ['OMP_NUM_THREADS'] = '1' #print(out_path) if not os.path.exists(out_path): os.makedirs(out_path) process_wrapper(md_path, fn, ncpu, chunksize)
def __init__(self, _favorites, _resumepoints): """Constructor for the ApiHelper class""" self._favorites = _favorites self._resumepoints = _resumepoints self._metadata = Metadata(_favorites, _resumepoints) install_opener(build_opener(ProxyHandler(get_proxies())))