Example #1
0
 def build(self, output_dir=""):
     """
     :type output_dir: path
     """
     self.dedup()
     for separator in self.separators:
         self.split(separator)
     self.root.export_source_tree()
     if not output_dir:
         return
     elif isdir(output_dir):
         print("Error: Output directory {} already exists.".format(
             output_dir))
         return
     for node in self.root.iter_dfs():
         if node.orig_file == "__VIRTUAL_TOP__":
             continue
         elif node.orig_file == self.root.childs[0][1].orig_file:
             target_dir = opjoin(output_dir, 'CONSTRAINT')
             is_top = True
         else:
             target_dir = opjoin(output_dir,
                                 self.mapping[dirname(node.orig_file)])
             is_top = False
         node.build(target_dir=target_dir, is_top=is_top)
Example #2
0
    def init_cam_vari(self, ip, port):
        self.npix = THERMAL_WIDTH * THERMAL_HEIGHT
        self.ip = ip
        self.port = port
        dll_path = opjoin('dll', 'CG_ThermalCamDll_2018.dll')
        self.dll = windll.LoadLibrary(dll_path)
        self.dll_2015 = windll.LoadLibrary(
            opjoin('dll', 'ThermalCamDll_2015.dll'))
        self.mHandle = wintypes.HANDLE()
        self.keepAlive = c_uint()
        self.camData = td.IRF_IR_CAM_DATA_T()
        #self.ushort_ptr = (c_ushort *(THERMAL_WIDTH * THERMAL_HEIGHT ))()
        #self.camData.ir_image = cast(self.ushort_ptr, POINTER(c_ushort))
        self.m16 = np.zeros((THERMAL_HEIGHT, THERMAL_WIDTH), dtype=np.uint16)
        self.acm32 = np.zeros((THERMAL_HEIGHT, THERMAL_WIDTH), dtype=np.uint32)
        self.camData.ir_image = self.m16.ctypes.data_as(POINTER(c_ushort))

        self.camData.image_buffer_size = 4 * THERMAL_WIDTH * THERMAL_HEIGHT
        self.lpsize = (c_byte * 8192)()
        self.camData.lpNextData = cast(self.lpsize, POINTER(c_byte))
        self.camData.dwSize = 0
        self.camData.dwPosition = 0
        if COX_MODEL == 'CG':
            self.corrPara = td.IRF_TEMP_CORRECTION_PAR_T_CG()
        else:
            self.corrPara = td.IRF_TEMP_CORRECTION_PAR_T()
        self.corrPara.atmTemp = 25.0
        self.corrPara.atmTrans = 1.0
        self.corrPara.emissivity = 1.0
        self.pfloat_lut = (c_float * 65536)()
        self.dll.SendCameraMessage.restype = c_short
        self.dll.SendCameraMessage.argtypes = [
            wintypes.HANDLE,
            POINTER(c_uint), c_int, c_ushort, c_ushort
        ]
Example #3
0
def build_logging(loglevel: str, logfile: str, message_format: str,
                  time_format: str):
    if logfile:
        logpath = opjoin(PROJECT_DIR, 'logs')
        if not os.path.exists(logpath):
            os.mkdir(logpath)
        path = opjoin(logpath, logfile)
        loghandler = RotatingFileHandler(path,
                                         mode='a',
                                         maxBytes=10 * 1024 * 1024,
                                         backupCount=10,
                                         encoding=None,
                                         delay=0)
        loghandler.setFormatter(logging.Formatter(message_format, time_format))
        loghandler.setLevel(loglevel)

        logger = logging.getLogger()
        logger.setLevel(loglevel)
        logger.addHandler(loghandler)
    else:
        logging.basicConfig(
            format=message_format,
            datefmt=time_format,
            level=loglevel,
        )
Example #4
0
def set_time_interval(from_day: float, to_day: float):
    """
    Sets start time and stop time to param.in file.
    :param from_day:
    :param to_day:
    :return:
    """
    def _edit_file(filepath: str, callback: Callable[[Iterable[str], TextIO], None]):
        with open(filepath) as f:
            out_fname = filepath + ".tmp"
            out = open(out_fname, "w")
            callback(f, out)
            out.close()
            os.rename(out_fname, filepath)

    def _update_params(infile: Iterable[str], outfile: TextIO):
        startday_pattern = ' start time (days)= '
        stopday_pattern = ' stop time (days) = '
        for line in infile:
            if line.startswith(startday_pattern):
                line = '%s%f\n' % (startday_pattern, from_day)
            if line.startswith(stopday_pattern):
                line = '%s%f\n' % (stopday_pattern, to_day)
            outfile.write(line)

    integrator_path = opjoin(Config.get_project_dir(), CONFIG['integrator']['dir'])
    param_in_filepath = opjoin(integrator_path, CONFIG.INTEGRATOR_PARAM_FILENAME)

    _edit_file(param_in_filepath, _update_params)
Example #5
0
    def install (self, opts = []):
        if rutils.is_true(self.get('BUILD')):
            res = super(CppUnitProject, self).install(opts = opts)
        else:
            res = []

        res += self.install_files(self.rc_path, self.install_rc_path, ['.*'])

        if self.get_lower(self.test_run_var_name) == 'yes':
            if self.type == 'shared':
                raise CppUnitError(self, "Cannot run a shared CppUnit test.")
            run_env = self.env.Clone()

            dirs = racy.renv.dirs
            install_bin = opjoin(dirs.install,"bin")
            install_lib = opjoin(dirs.install,"lib")

            run_env.Append(ENV = os.environ)
            run_env.AppendENVPath(racy.renv.LD_VAR, install_lib)
            execpath = opjoin(install_bin, self.full_name)
            run_test = run_env.Alias('run-'+self.name, res, execpath)
            if res:
                run_env.Depends(run_test, res[0])
            run_env.AlwaysBuild(run_test)

            res += run_test

        return res
Example #6
0
def clean():
    image_names = read_image_dir()

    removed = 0
    for image_name in image_names:
        basename, ext = get_base_ext(image_name)

        if len(ext) < 2 or ext[1] not in ['jpg', 'jpeg', 'png']:
            os.remove(opjoin(image_dir, image_name))
            removed += 1
            continue

        # Try to open
        try:
            im = Image.open(opjoin(image_dir, image_name))

            # Check resolution
            w, h = im.size
            if min(w, h) < args.clean:
                removed += 1
                im.close()
                os.remove(opjoin(image_dir, image_name))
                continue

            # Convert to RGB, save as JPEG
            im = im.convert('RGB')
            im.save(opjoin(image_dir, image_name), "JPEG")
        except:
            os.remove(opjoin(image_dir, image_name))
            removed += 1
    print(f'Removed {removed} images during cleaning process')
Example #7
0
    def session_info(self, where, name):
        """
		Get information on the specified session

		If where is '', will auto try the webscarab temp dir
		If where is not '', will try session nicknames 
		@param where: directory name where the sessions are located
		@param name: specific name of subdirectory containing desired session

		@return: C{dict} containing id, domains, transactions, date, seentids
			and whether the session is currently active
		"""
        if not where: where = self.get_tmpdir()
        if not opexists(opjoin(where, name)):
            wstmp = 'webscarab%s.tmp' % name
            if not opexists(opjoin(where, wstmp)):
                return {}
            name = wstmp
        session = {}
        wsdir = opjoin(where, name)
        session['id'] = wsdir
        m = re.search(r'webscarab(\d+).tmp', wsdir)
        if m:
            session['nickname'] = m.group(1)
        elif wsdir.rfind(os.sep) > -1:
            session['nickname'] = wsdir[wsdir.rfind(os.sep):]
        session['domains'] = self.domains_in_dir(wsdir)
        session['transactions'] = self.transactions(wsdir)
        session['date'] = os.path.getctime(wsdir)
        session['seentids'] = []
        if os.path.exists(opjoin(wsdir, 'conversationlog')):
            session['active'] = False
        else:
            session['active'] = True
        return session
Example #8
0
def mc2reco(fi,version="v5r4p0",newpath=""):
    """ converts allGamma-vXrYpZ_100GeV_10TeV-p2.noOrb.740485.mc.root to allGamma-v5r4p0_100GeV_10TeV-p2.noOrb.740485.reco.root"""
    #print '*** DEBUG: file: ',fi
    vtag = research("v\dr\dp\d",fi)
    if vtag is None:
        vtag = research("r\d{4}",fi)
    vtag = vtag.group(0)
    # lastly, replace the path
    if fi.startswith("root:"):
        #print fi
        fi = ("/%s"%fi.split("//")[2])
    fname = basename(fi)
    path = dirname(fi)
    task = basename(path)
    npath = opjoin(newpath,task)
    fout = opjoin(npath,fname)
    fout = fout.replace(".mc",".reco")
    max_occ = 10 # version tag shouldn't be there more than 10 times;
    # if we do not include this criterion, if MC-version == reco version this would yield an infinite loop!
    occ = 0
    while vtag in fout:
        fout = fout.replace(vtag,version)
        occ+=1
        if occ >= max_occ: break
    #print "*** DBG ***: ",fout
    return fout
Example #9
0
def set_time_interval(from_day: float, to_day: float):
    """
    Sets start time and stop time to param.in file.
    :param from_day:
    :param to_day:
    :return:
    """
    def _edit_file(filepath: str, callback: Callable[[Iterable[str], TextIO],
                                                     None]):
        with open(filepath) as f:
            out_fname = filepath + ".tmp"
            out = open(out_fname, "w")
            callback(f, out)
            out.close()
            os.rename(out_fname, filepath)

    def _update_params(infile: Iterable[str], outfile: TextIO):
        startday_pattern = ' start time (days)= '
        stopday_pattern = ' stop time (days) = '
        for line in infile:
            if line.startswith(startday_pattern):
                line = '%s%f\n' % (startday_pattern, from_day)
            if line.startswith(stopday_pattern):
                line = '%s%f\n' % (stopday_pattern, to_day)
            outfile.write(line)

    integrator_path = opjoin(Config.get_project_dir(),
                             CONFIG['integrator']['dir'])
    param_in_filepath = opjoin(integrator_path,
                               CONFIG.INTEGRATOR_PARAM_FILENAME)

    _edit_file(param_in_filepath, _update_params)
 def load_np(self, scan_id):
     path2scan = opjoin(self.augmented_data_path,
                        'generated_{}.mhd.npy'.format(scan_id))
     scan = np.load(path2scan)
     mask, origin, spacing, is_flip = load_itk_image(
         opjoin(segment_path, '{}.mhd'.format(scan_id)))
     return scan, origin, spacing, is_flip, mask, scan_id
Example #11
0
def _make_res(by_resonance: ResonanceMixin, filepaths: List[str],
              planets: tuple, integers: List[str]):
    asteroid_name = by_resonance.small_body.name
    resonance_id = by_resonance.id
    phase_storage = PhaseStorage.file
    phase_builder = PhaseBuilder(phase_storage)
    phase_loader = PhaseLoader(phase_storage)
    phase_cleaner = PhaseCleaner(phase_storage)

    print('Loading aei files.')
    builder = FilepathBuilder(filepaths, True)
    planet_aei_paths = [builder.build('%s.aei' % x) for x in planets]
    resmaker = ResfileMaker(planets, planet_aei_paths)
    getter = AEIDataGetter(builder)
    orbital_element_sets = build_bigbody_elements(planet_aei_paths)
    orbital_elem_set_facade = ResonanceOrbitalElementSetFacade(orbital_element_sets, by_resonance)

    aei_data = getter.get_aei_matrix(asteroid_name)
    phase_builder.build(aei_data, resonance_id, orbital_elem_set_facade)
    phases = phase_loader.load(resonance_id)

    folder = opjoin(getcwd(), 'res')
    if not exists(folder):
        mkdir(folder)
    resmaker.make(phases, aei_data, opjoin(
        folder, '%s_%s_%s.res' % (asteroid_name, '_'.join(planets),
                                  '_'.join([str(x) for x in integers]))
    ))
    phase_cleaner.delete(resonance_id)
Example #12
0
def test_filepathbuilder(aei_fixture, is_recursive: bool):
    builder = FilepathBuilder(aei_fixture, is_recursive)
    if not is_recursive:
        with pytest.raises(FilepathException):
            assert builder.build(TEST_FILE) == opjoin(SUB_FIXTURES, TEST_FILE)
    else:
        assert builder.build(TEST_FILE) == opjoin(SUB_FIXTURES, TEST_FILE)
    assert builder.build(TEST2_FILE) == opjoin(TEMP_FIXTURES, TEST2_FILE)
Example #13
0
def test_filepathbuilder(aei_fixture, is_recursive: bool):
    builder = FilepathBuilder(aei_fixture, is_recursive)
    if not is_recursive:
        with pytest.raises(FilepathException):
            assert builder.build(TEST_FILE) == opjoin(SUB_FIXTURES, TEST_FILE)
    else:
        assert builder.build(TEST_FILE) == opjoin(SUB_FIXTURES, TEST_FILE)
    assert builder.build(TEST2_FILE) == opjoin(TEMP_FIXTURES, TEST2_FILE)
    def load(self, subfolder=None):
        self.baseline_mc = FROCMetricsCalculator(label=self.baseline_label)
        subfolder = '' if subfolder is None else subfolder
        baseline_load_path = opjoin(subfolder, self.baseline_label)
        self.baseline_mc.load(baseline_load_path + '.npy')

        self.augmented_mc = FROCMetricsCalculator(label=self.augmented_label)
        aug_load_path = opjoin(subfolder, self.augmented_label)
        self.augmented_mc.load(aug_load_path + '.npy')
Example #15
0
    def ready(self):
        # Create directories
        os.makedirs(opjoin(settings.TRAINING_DIR, 'ckpts'), exist_ok=True)
        os.makedirs(opjoin(settings.TRAINING_DIR, 'logs'), exist_ok=True)
        os.makedirs(opjoin(settings.INFERENCE_DIR, 'ckpts'), exist_ok=True)
        os.makedirs(opjoin(settings.INFERENCE_DIR, 'logs'), exist_ok=True)

        os.makedirs(settings.DATASETS_DIR, exist_ok=True)
        os.makedirs(settings.OUTPUTS_DIR, exist_ok=True)
Example #16
0
 def save_h(self):
     path=opjoin('cfg','x_h.cfg')
     fid=open(path,'w')
     fid.write('%.3f'%self.x_h)
     fid.close()
     path=opjoin('cfg','y_h.cfg')
     fid=open(path,'w')
     fid.write('%.3f'%self.y_h)
     fid.close()
Example #17
0
class SmallBodiesFileBuilderTestCase(unittest.TestCase):
    FILEPATH = opjoin(PROJECT_DIR, 'small.in')
    SYMLINK = opjoin(PROJECT_DIR, 'small.in.link')

    def tearDown(self):
        if opexists(self.FILEPATH):
            os.remove(self.FILEPATH)

    def test_create_small_body_file(self):
        from resonances.integrator import SmallBodiesFileBuilder

        def _test_with_symlink():
            builder = SmallBodiesFileBuilder(self.FILEPATH, self.SYMLINK)
            builder.create_small_body_file()

            self.assertTrue(opexists(self.FILEPATH))
            self.assertTrue(opexists(self.SYMLINK))

            os.remove(self.FILEPATH)
            os.remove(self.SYMLINK)

        def _test_without_symlink():
            builder = SmallBodiesFileBuilder(self.FILEPATH)
            builder.create_small_body_file()

            self.assertTrue(opexists(self.FILEPATH))
            self.assertFalse(opexists(self.SYMLINK))

            os.remove(self.FILEPATH)

        _test_with_symlink()
        _test_without_symlink()

    def test_flush(self):
        from resonances.integrator import SmallBodiesFileBuilder

        def _test_file_existance():
            builder = SmallBodiesFileBuilder(self.FILEPATH)
            builder.add_body(1, [1., 2., 3., 4., 5., 6., 7.])
            with self.assertRaises(FileNotFoundError):
                builder.flush()

        def _test_success_flush():
            builder = SmallBodiesFileBuilder(self.FILEPATH)
            builder.create_small_body_file()
            builder.add_body(1, [1., 2., 3., 4., 5., 6., 7.])
            builder.flush()

            with open(self.FILEPATH) as f:
                total_lines_count = sum(1 for _ in f)
                self.assertEqual(total_lines_count, 7)

            os.remove(self.FILEPATH)

        _test_file_existance()
        _test_success_flush()
Example #18
0
def get_pyg_dataset(dataroot, dataset):
    if dataset in ['cora', 'citeseer', 'pubmed']:
        graph = pyg.datasets.Planetoid(root=opjoin(dataroot, dataset),
                                       name=dataset.capitalize())
    elif dataset == 'coauthorcs':
        graph = pyg.datasets.Coauthor(root=opjoin(dataroot, dataset),
                                      name='CS')
    else:
        raise NotImplementedError
    return graph
Example #19
0
 def load_bbody_emv(self):
     path=opjoin('cfg','bbody_emv_h.cfg')
     fid=open(path,'r')
     self.bbody_emv=[0,0]
     self.bbody_emv[0]=float(fid.read())
     fid.close()
     path=opjoin('cfg','bbody_emv_l.cfg')
     fid=open(path,'r')
     self.bbody_emv[1]=float(fid.read())
     fid.close()
Example #20
0
 def runner_src(self):
     sources = {
         'shared':'testRunnerShared.cpp',
         'exec'  :'testRunnerExec.cpp'  ,
         'xml'   :'testRunnerXML.cpp'   ,
         }
     runner = sources.get(self.test_type)
     runner = [opjoin(self.runner_build_dir, runner)]
     if self.associated_prj.is_bundle:
         runner += [ opjoin(self.runner_build_dir, 'testBundle.cpp') ]
     return runner
Example #21
0
def create_index(data_path):
    ids = []
    for root, _, files in os.walk(data_path):
        if glob.glob(opjoin(data_path, root, '*xml')):
            nodules = parseXML(opjoin(data_path, root))
            id2roi = create_map_from_nodules(nodules)
            if len(id2roi) == 0:
                continue
            ids.append(root)
    with open('index.json', 'w') as write_file:
        json.dump(ids, write_file)
def select_brand():
    mappings = get_all_brands(return_type=dict)
    targets_path = get_targets_path()
    msg = ""
    for k, v in mappings.items():
        msg += f"{k}: {v}\n"
    select = input(f"Please Input 'Crawler Number' Or 'all' \n{msg}")
    if select != "all":
        return opjoin(targets_path, mappings[select])
    elif select == "all":
        return list(map(lambda c: opjoin(targets_path, c), mappings.values()))
Example #23
0
    def __init__(self, output_dir: str):
        outoption = CONFIG['output']
        self.output_dir = output_dir
        self.output_images = opjoin(output_dir, outoption['images'])
        self.output_res_path = opjoin(output_dir, outoption['angle'])
        self.output_gnu_path = opjoin(output_dir, outoption['gnuplot'])

        if not os.path.exists(self.output_images):
            os.makedirs(self.output_images)

        if not os.path.exists(self.output_gnu_path):
            os.makedirs(self.output_gnu_path)
Example #24
0
    def save_npy(self, start, end):
        for i in range(start, end):
            print('processing %d' % i)
            try:
                imgs, bbox = self.get_data_from_dcm(i)
            except:
                imgs, bbox = self.get_data_from_dcm(i - 1)

            save_imgs_path = opjoin(self.lidc_npy_path, 'imgs_%d.npy' % i)
            save_bbox_path = opjoin(self.lidc_npy_path, 'bbox_%d.npy' % i)

            np.save(save_bbox_path, bbox)
            np.save(save_imgs_path, imgs)
Example #25
0
def extract_trans(t):
    if not 199 < int(t['code']) < 300: return
    if not 'respbody' in t: return
    if not t['respbody']: return
    try:
        if not opexists(Extract): os.mkdir(Extract)
        pdir = opjoin(Extract, t['hostname'] + '_' + t['port'])
        if not opexists(pdir):
            os.mkdir(pdir)
        os.makedirs(opjoin(pdir, t['dir'][1:]))
    except OSError, e:
        if e.errno != 17:
            log.error('Error: %s' % e.strerror)
Example #26
0
 def target_file(self):
     """
     Query command of target file name.
     """
     if self.stage:
         segments = basename(self.orig_file).split(".")
         segments.insert(-1, self.stage)
         target_file = opjoin(dirname(self.orig_file), ".".join(segments))
     else:
         target_file = self.orig_file
     if self.target_dir:
         target_file = opjoin(self.target_dir, basename(target_file))
     return target_file
Example #27
0
def test_make_plot(asteroid_number):
    resfilepath = opjoin(PROJECT_DIR, 'tests', 'fixtures',
                         'A%s.res' % asteroid_number)
    gnufilepath = opjoin(PROJECT_DIR, 'tests', 'fixtures',
                         'A%s.gnu' % asteroid_number)
    pngfilepath = opjoin(PROJECT_DIR, 'tests', 'fixtures',
                         'A%s.png' % asteroid_number)
    make_plot(resfilepath, gnufilepath, pngfilepath)

    assert os.path.exists(pngfilepath)
    assert os.path.exists(gnufilepath)
    os.remove(pngfilepath)
    os.remove(gnufilepath)
Example #28
0
 def save_thd(self):
     if COX_MODEL == 'CG':
         fname = opjoin('cfg', 'thd_cg.cfg')
     else:
         fname = opjoin('cfg', 'thd.cfg')
     fid = open(fname, 'w')
     self.change_thd(self.cfg_data.thd_cels)
     fid.write('%d' % self.cfg_data.thd)
     fid.close()
     path = opjoin('cfg', 'thd_cels.cfg')
     fid = open(path, 'w')
     fid.write('%.1f' % self.cfg_data.thd_cels)
     fid.close()
Example #29
0
 def get(self, session, tinfo):
     t = {}
     t['id'] = tinfo['id']
     t['source'] = self.proxy_name
     t['request'] = open(
         opjoin(session['id'], 'conversations', tinfo['id'] + '-request'),
         'rb').read()
     t['response'] = open(
         opjoin(session['id'], 'conversations', tinfo['id'] + '-response'),
         'rb').read()
     # XXX: this needs to verify that the transaction are actually complete
     session['seentids'].append(tinfo['id'])
     return t
Example #30
0
def extract_trans(t):
	if not 199 < int(t['code']) < 300: return
	if not 'respbody' in t: return
	if not t['respbody']: return
	try:
		if not opexists(Extract): os.mkdir(Extract)
		pdir = opjoin(Extract, t['hostname']+'_'+t['port'])
		if not opexists(pdir):
			os.mkdir(pdir)
		os.makedirs(opjoin(pdir, t['dir'][1:]))
	except OSError, e:
		if e.errno != 17:
			log.error('Error: %s' % e.strerror)
Example #31
0
def get_files_with_nodules(nodules, root):
    files = os.listdir(root)
    image_ids_with_nodules = set()
    for nodule in nodules:
        for roi in nodule['roi']:
            image_ids_with_nodules.add(roi['sop_uid'])
    result = []
    for file in files:
        if not file.endswith('dcm'):
            continue
        _, ds = imread(opjoin(root, file))
        if ds.SOPInstanceUID in image_ids_with_nodules:
            result.append(opjoin(root, file))
    return result
Example #32
0
def main():
    gfpath = sys.argv[1]
    dirpath = sys.argv[2]

    with open(gfpath, 'r') as f:
        lines = [line.split(',') for line in f]

    for line in lines:
        if int(line[1]) > 4:
            if not os.path.exists(opjoin('temp_forms', line[0])):
                os.makedirs(opjoin('temp_forms', line[0]))
                lst = [
                    f for f in os.listdir(opjoin(dirpath, line[0]))
                    if '.jpg' in f
                ]
                shuffle(lst)
                for f in lst[:5]:
                    # print(opjoin(dirpath, line[0], f))
                    copyfile(opjoin(dirpath, line[0], f),
                             opjoin('temp_forms', line[0], f))

                for temp in os.listdir(opjoin(dirpath, line[0])):
                    if 'template' in temp:
                        copyfile(opjoin(dirpath, line[0], temp),
                                 opjoin('temp_forms/trains', line[0], temp))

                print(line[0])
Example #33
0
    def _build_from_dirs(self, for_name: str) -> str:
        res = None
        if self._is_recursive:
            for path_base in self._dirs:
                for filepath in glob.iglob(opjoin(path_base, '**', for_name), recursive=True):
                    res = filepath
                    break
        else:
            for path_base in self._dirs:
                filepath = opjoin(path_base, for_name)
                if opexists(filepath):
                    res = filepath

        return res
Example #34
0
 def rel_file(self):
     """
     Query command of relative file name. Relative file name is used when
     rendering script names in source commands.
     """
     if self.stage:
         segments = basename(self.orig_file).split(".")
         segments.insert(-1, self.stage)
         rel_file = opjoin(dirname(self.orig_file), ".".join(segments))
     else:
         rel_file = self.orig_file
     if self.target_dir:
         rel_file = opjoin(self.rel_dir, basename(rel_file))
     return rel_file
Example #35
0
 def install_nodes(self):
     prj = self
     env = self.env
     initmodel = opjoin(prj.rc_path,'__init__.py')
     if os.path.isfile(initmodel):
         content  = rutils.get_file_content(initmodel)
         initfile = opjoin(prj.local_dir, '__init__.py')
         write    = prj.WriteBuilder(initfile, content)
         copy     = prj.CopyBuilder(
                 '${LOCAL_DIR}',
                 prj.install_pkg_path,
                 marker_name='InstallPkgs'
                 )
         env.Depends(copy, write)
     return [copy, write]
Example #36
0
    def _build_from_dirs(self, for_name: str) -> str:
        res = None
        if self._is_recursive:
            for path_base in self._dirs:
                for filepath in glob.iglob(opjoin(path_base, '**', for_name),
                                           recursive=True):
                    res = filepath
                    break
        else:
            for path_base in self._dirs:
                filepath = opjoin(path_base, for_name)
                if opexists(filepath):
                    res = filepath

        return res
Example #37
0
    def get(self, request, *args, **kwargs):
        """Retrieve results about an inference process

        This API provides information about an `inference` process.In classification task it returns the list \
        of images and an array composed of the classes prediction scores.
        In segmentation task it returns the URLs of the segmented images.
        """
        if not self.request.query_params.get('process_id'):
            error = {'Error': f'Missing required parameter `process_id`'}
            return Response(data=error, status=status.HTTP_400_BAD_REQUEST)
        process_id = self.request.query_params.get('process_id')
        infer = models.Inference.objects.filter(celery_id=process_id)
        if not infer:
            # already deleted weight/training or inference
            return Response(
                {
                    "result":
                    "Process stopped before finishing or non existing."
                },
                status=status.HTTP_404_NOT_FOUND)

        if AsyncResult(process_id).status == 'PENDING':
            return Response(
                {
                    "result":
                    "Process in execution. Try later for output results."
                },
                status=status.HTTP_200_OK)

        infer = infer.first()
        if not os.path.exists(opjoin(settings.OUTPUTS_DIR, infer.outputfile)):
            return Response({"result": "Output file not found"},
                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
        outputs = open(opjoin(settings.OUTPUTS_DIR, infer.outputfile), 'r')
        # Differentiate classification and segmentation
        if infer.modelweights_id.model_id.task_id.name.lower(
        ) == 'classification':
            lines = outputs.read().splitlines()
            lines = [line.split(';') for line in lines]
            # preds = self.trunc(preds, decs=8)
        else:
            # Segmentation
            # output file contains path of files
            uri = request.build_absolute_uri(settings.MEDIA_URL)
            lines = outputs.read().splitlines()
            lines = [l.replace(settings.OUTPUTS_DIR, uri) for l in lines]
        response = {'outputs': lines}
        return Response(response, status=status.HTTP_200_OK)
Example #38
0
def genres(asteroid_number: int, integers: List[int], filepaths: List[str], planets: Tuple):
    t1 = aliased(Planet)
    t2 = aliased(Planet)
    cond = len(integers) == 3
    resonance_cls = ThreeBodyResonance if cond else TwoBodyResonance
    query = (
        session.query(resonance_cls).outerjoin(t1, t1.id == resonance_cls.first_body_id).filter(t1.name == planets[0])
    )
    if cond:
        query = query.outerjoin(t2, t2.id == resonance_cls.second_body_id).filter(t2.name == planets[1])
    query = query.outerjoin(Asteroid, Asteroid.id == resonance_cls.small_body_id).filter(
        Asteroid.name == "A%i" % asteroid_number
    )

    resonance = query.first()
    if not resonance:
        logging.warning("There is no resonance by pointed filter.")
        return
    resonance_id = resonance.id

    phase_storage = PhaseStorage.file
    phase_builder = PhaseBuilder(phase_storage)
    phase_loader = PhaseLoader(phase_storage)
    phase_cleaner = PhaseCleaner(phase_storage)

    print("Loading aei files.")
    builder = FilepathBuilder(filepaths, True)
    planet_aei_paths = [builder.build("%s.aei" % x) for x in planets]
    resmaker = ResfileMaker(planets, planet_aei_paths)
    getter = AEIDataGetter(builder)
    orbital_element_sets = build_bigbody_elements(planet_aei_paths)
    orbital_elem_set_facade = ResonanceOrbitalElementSetFacade(orbital_element_sets, resonance)

    aei_data = getter.get_aei_data(asteroid_number)
    phase_builder.build(aei_data, resonance_id, orbital_elem_set_facade)
    phases = phase_loader.load(resonance_id)

    folder = opjoin(getcwd(), "res")
    if not exists(folder):
        mkdir(folder)
    resmaker.make(
        phases,
        aei_data,
        opjoin(folder, "A%i_%s_%s.res" % (asteroid_number, "_".join(planets), "_".join([str(x) for x in integers]))),
    )
    phase_cleaner.delete(resonance_id)
Example #39
0
def aei_fixture(request):
    os.mkdir(TEMP_FIXTURES)
    os.mkdir(SUB_FIXTURES)
    test_filepaths = [opjoin(SUB_FIXTURES, TEST_FILE), opjoin(TEMP_FIXTURES, TEST2_FILE)]

    for path in test_filepaths:
        with open(path, 'w') as test_file:
            test_file.write('123')

    paths = (FIXTURES, TEMP_FIXTURES)

    def fin():
        for item in test_filepaths:
            os.remove(item)
        os.rmdir(TEMP_FIXTURES)
        os.rmdir(SUB_FIXTURES)

    request.addfinalizer(fin)
    return paths
Example #40
0
def load_resonances(start: int, stop: int, file: str, axis_swing: float,
                    planets: Tuple[str]):
    assert axis_swing > 0.
    from commands import load_resonances as _load_resonances
    if not os.path.isabs(file):
        file = os.path.normpath(opjoin(os.getcwd(), file))
    if file == RESONANCE_FILEPATH:
        logging.info('%s will be used as source of integers' % file)
    for i in range(start, stop, STEP):
        end = i + STEP if i + STEP < stop else stop
        _load_resonances(file, i, end, planets, axis_swing)
Example #41
0
def simple_clean(with_aei=True):
    """Execute simple_clean.sh

    :param with_aei:
    :rtype bool:
    """
    for ext in EXTENSIONS:
        for filename in glob(opjoin(INTEGRATOR_DIR, '*.%s' % ext)):
            os.remove(filename)
    if with_aei:
        aei_clean()
Example #42
0
def gimme(root, wanted):
    try:
        for f in find(root, wanted):
            yield opjoin(f + '.sdc', 'Data.h5')
        return
    except ValueError, e:
        if not str(e).startswith('not a directory'):
            raise
        with open(root) as h:
            for line in h:
                yield line.rstrip('\n')
Example #43
0
 def _build_from_tars(self, for_name: str) -> str:
     archives = [self._last_tar] + self._archives if self._last_tar else self._archives
     for tarname in archives:
         with taropen(tarname) as tarfile:  # type: TarFile
             for taritem in tarfile:  # type: TarInfo
                 filepath = taritem.name
                 if for_name not in filepath:
                     continue
                 tarfile.extract(taritem, EXTRACT_PATH)
                 self._last_tar = tarname
                 return opjoin(EXTRACT_PATH, filepath)
     return None
Example #44
0
    def create_install_targets(self,list_dir):
        # list targets = [(dir, list_targets),...]
        list_targets = []
        install_dir = racy.renv.dirs.install

        for tdir in list_dir:
            dir_path = opjoin(install_dir,tdir)

            if os.path.exists(dir_path):
                targets = self.create_targets(dir_path)
                list_targets.append((tdir,targets)) 

        return list_targets
Example #45
0
    def result (self, deps_results):
        res = super(CppUnitProject, self).result(deps_results=deps_results)
        if self.associated_prj.is_bundle:
            env = self.env
            if self.type == 'shared':
                raise CppUnitError(self,
                        "Cannot build a bundle test as a shared library.")
            prj_version = r'\"{0.version}\"'.format(self)
            env.AppendUnique(CPPDEFINES=('CPPUNIT_TEST_VERSION', prj_version))
            bundle_test_header = opjoin(self.runner_src_path, 'testBundle.hpp')
            env['FORCE_INCLUDE'] = bundle_test_header
            env.Depends( res, env.File(bundle_test_header) )


        return res
Example #46
0
def test_orbital_elements():
    filepath = opjoin(PROJECT_DIR, PARAMS['integrator']['dir'], 'A1.aei')
    collection = OrbitalElementSetCollection(filepath)
    assert len(collection.orbital_elements) == A1_AEI_FILE_LEN
    assert len(collection) == A1_AEI_FILE_LEN
    orbitalelements = collection.orbital_elements[0]

    assert orbitalelements == collection[0]
    assert orbitalelements.time == 0.0000000
    assert orbitalelements.p_longitude == radians(1.541309E+02)
    assert orbitalelements.mean_anomaly == radians(3.172742E+02)
    assert orbitalelements.semi_axis == 2.76503
    assert orbitalelements.eccentricity == 0.077237
    assert orbitalelements.inclination == radians(int(10.6047))
    assert orbitalelements.node == radians(int(80.4757))
Example #47
0
def remove_export_directory(start: int, stop: int=None) -> bool:
    """Removes directory, that created by method package.

    :param start:
    :param stop:
    :return: flag says about succesful of operation
    """
    if not stop:
        stop = start + BODIES_COUNTER
    export_dir = opjoin(EXPORT_BASE_DIR, '%i-%i' % (start, stop))
    if os.path.exists(export_dir):
        logging.info('Clear directory %s...' % export_dir)
        shutil.rmtree(export_dir)
        logging_done()
        return True
    else:
        logging.info('Nothing to delete')
        return False
Example #48
0
def find(root, wanted=None, sortkey=None):
    if not os.path.isdir(root):
        raise ValueError('not a directory: %s' % root)
    for r, ds, fs in os.walk(root):

        if sortkey:
            ks = sortkey
        else:
            ks = (None, None)

        for seq, sk in zip((ds, fs), ks):
            if sk:
                seq.sort(key=sk)
            else:
                seq.sort()

        for s in (ds, fs):
            for i in s:
                if wanted is None or wanted(i, r, isdir=(s==ds)):
                    yield opjoin(r, i)
Example #49
0
def _get_from_s3(filepaths: List[str]) -> List[str]:
    new_paths = []
    if any([is_s3(x) for x in filepaths]):
        conn = S3Connection(CONFIG['s3']['access_key'], CONFIG['s3']['secret_key'])
        bucket = conn.get_bucket(BUCKET)
        for path in filepaths:
            if not is_s3(path):
                continue
            start = path.index(BUCKET)
            filename = path[start + len(BUCKET) + 1:]
            if not opexists(S3_FILES_DIR):
                makedirs(S3_FILES_DIR)
            local_path = opjoin(S3_FILES_DIR, basename(filename))
            if not opexists(local_path):
                s3key = bucket.get_key(filename, validate=False)
                with open(local_path, 'wb') as f:
                    s3key.get_contents_to_file(f)
                if not is_tarfile(local_path):
                    raise FilepathInvalidException('%s is not tar. Local copy %s' %
                                                   (path, local_path))
            new_paths.append(local_path)
    return new_paths
Example #50
0
	if not 'respbody' in t: return
	if not t['respbody']: return
	try:
		if not opexists(Extract): os.mkdir(Extract)
		pdir = opjoin(Extract, t['hostname']+'_'+t['port'])
		if not opexists(pdir):
			os.mkdir(pdir)
		os.makedirs(opjoin(pdir, t['dir'][1:]))
	except OSError, e:
		if e.errno != 17:
			log.error('Error: %s' % e.strerror)

	fn = t['file'].rsplit('.', 1)
	if len(fn[0]) > 30:
		fn[0] = fn[0][:30]
	outfn = opjoin(pdir, t['dir'][1:], fn[0].translate(FNTrans))
	outfn += '.'+ str(t['id'])
	if len(fn) > 1:
		if len(fn[1]) > 10:
			fn[1] = fn[1][:10]
		outfn += '.' + fn[1].translate(FNTrans)
	#log.info('extract_trans: writing to %s' % outfn)
	f = open(outfn, 'wb')
	f.write(t['respbody'])
	f.close()

def scan(wproxy, session, checks, pmd, urlfilter, hostfilter):
	"""
	Parse all transactions in a given directory

	@param wproxy: pmproxy subclass
Example #51
0
 def runner_build_dir(self):
     return opjoin(self.build_dir, 'CPPUnit_runner')
Example #52
0
 def runner_src_path(self):
     return opjoin(CPPUNIT_PLUGIN_PATH, 'rc')
Example #53
0
from os.path import join as opjoin
from shutil import copyfile
import os
from settings import Config

PROJECT_DIR = Config.get_project_dir()
INTEGRATOR_PATH = opjoin(PROJECT_DIR, Config.get_params()['integrator']['dir'])
PARAMS = Config.get_params()


def test_set_time_interval():
    from integrator import set_time_interval

    def _copyfile(name: str):
        path = opjoin(INTEGRATOR_PATH, name)
        target = opjoin(INTEGRATOR_PATH, name + '.backup')
        copyfile(path, target)
        return path

    param_in_filepath = _copyfile(PARAMS.INTEGRATOR_PARAM_FILENAME)

    set_time_interval(1, 2)

    startday_assert_flag = False
    stopday_assert_flag = False
    with open(param_in_filepath) as f:
        for line in f:
            startday_assert_flag = startday_assert_flag or (' start time (days)= 1' in line)
            stopday_assert_flag = stopday_assert_flag or (' stop time (days) = 2' in line)

    assert startday_assert_flag
Example #54
0
from os.path import exists as opexists
from os.path import join as opjoin
from os.path import isabs
from os import makedirs
import glob
from tarfile import is_tarfile
from tarfile import open as taropen
import os
from settings import Config
import shutil
from os.path import basename

PROJECT_DIR = Config.get_project_dir()
CONFIG = Config.get_params()
_ex_folder = CONFIG['extract_dir']
EXTRACT_PATH = _ex_folder if isabs(_ex_folder) else opjoin(PROJECT_DIR, _ex_folder)

BUCKET = CONFIG['s3']['bucket']
_s3_folder = CONFIG['s3files_dir']
S3_FILES_DIR = _s3_folder if isabs(_s3_folder) else opjoin(PROJECT_DIR, _s3_folder)


class FilepathException(Exception):
    pass


class FilepathInvalidException(Exception):
    pass


def _get_from_s3(filepaths: List[str]) -> List[str]:
Example #55
0
 def _copyfile(name: str):
     path = opjoin(INTEGRATOR_PATH, name)
     target = opjoin(INTEGRATOR_PATH, name + '.backup')
     copyfile(path, target)
     return path
Example #56
0
 def get_path (self, path = ""):
     """Returns <project>/test/[path]"""
     root = super(CppUnitProject, self).get_path(constants.TEST_PATH)
     path = opjoin(root, self.cppunit_test_dir, path)
     return os.path.abspath(os.path.normpath(path))
Example #57
0
 def get_options_file(prj):
     opt_file = opjoin(
                 prj.get_path(constants.TEST_PATH), 
                 CppUnitProject.cppunit_option_file
                 )
     return opt_file
Example #58
0
from .internal import asteroid_interval_options
from .internal import asteroid_time_intervals_options
from .internal import Path
from .internal import report_interval_options
from settings import Config
from os.path import join as opjoin

LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
PHASE_STORAGE = ['REDIS', 'DB', 'FILE']
PLANETS = ['EARTHMOO', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'PLUTO', 'SATURN', 'URANUS',
           'VENUS']

CONFIG = Config.get_params()
PROJECT_DIR = Config.get_project_dir()
RESONANCE_TABLE_FILE = CONFIG['resonance_table']['file']
RESONANCE_FILEPATH = opjoin(PROJECT_DIR, 'axis', RESONANCE_TABLE_FILE)
STEP = CONFIG['integrator']['number_of_bodies']
INTEGRATOR_DIR = CONFIG['integrator']['dir']

BodyCountType = TypeVar('T', str, int)


@click.group()
@click.option('--loglevel', default='DEBUG', help='default: DEBUG',
              type=click.Choice(LEVELS))
@click.option('--logfile', default=None, help='default: None',
              type=str)
def cli(loglevel: str = 'DEBUG', logfile: str = None):
    build_logging(getattr(logging, loglevel), logfile, '%(asctime)s %(levelname)s %(message)s',
                  '%Y-%m-%d %H:%M:%S')
Example #59
0
    def result(self, deps_results=True):
        prj = self
        env = self.env

        result = []

        class ConfigureMethods(object):
            prj = self
            for name, f in self.env_functions.items():
                locals()[name] = f

        prj.configure_env()
        prj.configure_consumer(ConfigureMethods, False)
        command = CommandWrapper(prj,'SysCommand')
        prj.prj_locals['generate']()

        if racy.renv.is_darwin():
            install_tool = opjoin(racy.get_bin_path(),'..','Utils',
                                    'osx_install_name_tool.py')
            libs = [self.lib_path]
            deps_lib = self.ENV['DEPS_LIB']
            if deps_lib:
                libs += deps_lib.split(':')

            command(['python', install_tool, '-i', '-a', '-P','*',
                '-s',self.build_bin_path] + libs , pwd = '.')


        #import defined strings and functions from generate method
        for k,v in self.ENV.items():
            if isinstance(v, basestring) or callable(v):
                env[k] = v

        res = [
                self.MkdirBuilder('${LOCAL_DIR}'),
                self.MkdirBuilder('${LOCAL_DIR}/bin'),
                self.MkdirBuilder('${LOCAL_DIR}/lib'),
                self.MkdirBuilder('${LOCAL_DIR}/include'),
                ]
        res += BuilderWrapper.apply_calls( prj, **self.ENV )

        downloads = [x for x in res if self.download_target in str(x)]
        map(res.remove, downloads)
        res = downloads + res

        previous_node = []
        for nodes in res:
            if not isinstance(nodes, LibextProject):
                for node in nodes:
                    #HACK: scons need a name attribute to manage dependencies
                    if not hasattr(node, "name"):
                        node.name = ''
                    env.Depends( node, previous_node )
                    previous_node = node
            elif deps_results:
                previous_node = [previous_node, nodes.deps_build_nodes]

        if not isinstance(nodes, LibextProject):
            result += nodes
        else:
            result += previous_node

        for node in [prj.extract_dir]:
            env.Clean(node, node)


        alias = 'result-{prj.type}-{prj.full_name}'
        result = env.Alias (alias.format(prj=self), result)
        return result