def measurement_block(self, block): sgroups, samples, mtypes, ftype = block.split('_') # names with , need to be replaced if not '(' in samples and ',' in samples: samples = samples.replace(',', '.') RockPy3.logger.warning('sample name %s contains \',\' will be replaced with \'.\'' % samples) self.sgroups, self.samples, self.mtypes, self.ftype = self.extract_tuple(sgroups), self.extract_tuple( samples), self.extract_tuple(mtypes), ftype self.mtypes = tuple(RockPy3.abbrev_to_classname(mtype) for mtype in RockPy3._to_tuple(self.mtypes)) self.ftype = RockPy3.abbrev_to_classname(ftype)
def measurement_block(self, block): sgroups, samples, mtypes, ftype = block.split('_') # names with , need to be replaced if not '(' in samples and ',' in samples: samples = samples.replace(',', '.') RockPy3.logger.warning( 'sample name %s contains \',\' will be replaced with \'.\'' % samples) self.sgroups, self.samples, self.mtypes, self.ftype = self.extract_tuple( sgroups), self.extract_tuple(samples), self.extract_tuple( mtypes), ftype self.mtypes = tuple( RockPy3.abbrev_to_classname(mtype) for mtype in RockPy3._to_tuple(self.mtypes)) self.ftype = RockPy3.abbrev_to_classname(ftype)
def measurement_infos(self): idict = {'fpath': self.fpath, 'ftype': self.ftype, 'idx': self.suffix, 'series': self.series} samples = RockPy3._to_tuple(self.samples) for i in samples: for j in self.mtypes: mtype = RockPy3.abbrev_to_classname(j) idict.update({'mtype': mtype, 'sample': i}) yield idict
def measurement_infos(self): idict = { 'fpath': self.fpath, 'ftype': self.ftype, 'idx': self.suffix, 'series': self.series } samples = RockPy3._to_tuple(self.samples) for i in samples: for j in self.mtypes: mtype = RockPy3.abbrev_to_classname(j) idict.update({'mtype': mtype, 'sample': i}) yield idict
def import_folder( self, folder, sname=None, sgroup=None, mtype=None, automatic_results=True, ): # todo specify samples, mtypes and series for selective import of folder """ imports all files in the specified folder Parameters ---------- folder gname Returns ------- """ files = [ os.path.join(folder, i) for i in os.listdir(folder) if not i.startswith('#') if not i.startswith(".") if not i.endswith("pynb") if not i.endswith("log") if not os.path.isdir(os.path.join(folder, i)) ] measurements = [] start = time.clock() minfos = (RockPy3.core.file_operations.minfo(f) for f in files) minfos = (minfo for minfo in minfos if minfo.is_readable()) if mtype: mtype = RockPy3._to_tuple(mtype) mtype = [RockPy3.abbrev_to_classname(mt) for mt in mtype] for mi in minfos: RockPy3.logger.info('importing {}'.format(mi.fpath)) # dont import samples that are not in the measurement info if sname: sname = RockPy3._to_tuple(sname) if not any(s in mi.samples for s in sname): continue # dont import samples that are not included in a certain samplegroup in the measurement info if sgroup: sgroup = RockPy3._to_tuple(sgroup) if not any(sg in mi.sgroups for sg in sgroup): continue # check if the filename has not been imported, yet if mi.fpath in self.imported_files: continue else: self.imported_files.append(mi.fpath) for sinfo in mi.sample_infos: s = self.add_sample(warnings=False, **sinfo) for minfo in mi.measurement_infos: if mtype and minfo['mtype'].lower() not in mtype: continue if minfo['sample'] == sinfo['name']: measurements.append(s.add_measurement(**minfo)) end = time.clock() print('IMPORT generated {} measurements: finished in {:.02e}s'.format( len(measurements), end - start)) return measurements
def get_measurement(self, mtype=None, series=None, stype=None, sval=None, sval_range=None, mean=False, invert=False, id=None, result=None ): """ Returns a list of measurements of type = mtypes Parameters ---------- mtypes: list, str mtypes to be returned series: list(tuple) list of tuples, to search for several sepcific series. e.g. [('mtime',4),('gc',2)] will only return mesurements that fulfill both criteria. stypes: list, str series type sval_range: list, str series range e.g. sval_range = [0,2] will give all from 0 to 2 including 0,2 also '<2', '<=2', '>2', and '>=2' are allowed. svals: float series value to be searched for. caution: will be overwritten when sval_range is given invert: if invert true it returns only measurements that do not meet criteria sval_range: can be used to look up measurements within a certain range. if only one value is given, it is assumed to be an upper limit and the range is set to [0, sval_range] mean: bool id: list(int) search for given measurement id Returns ------- if no arguments are passed all sample.measurements list of RockPy.Measurements that meet search criteria or if invert is True, do not meet criteria. [] if none are found Note ---- there is no connection between stype and sval. This may cause problems. I you have measurements with M1: [pressure, 0.0, GPa], [temperature, 100.0, C] M2: [pressure, 1.0, GPa], [temperature, 100.0, C] and you search for stypes=['pressure','temperature'], svals=[0,100]. It will return both M1 and M2 because both M1 and M2 have [temperature, 100.0, C]. """ if mean: mlist = self.mean_measurements else: mlist = self.measurements if id is not None: id = RockPy3._to_tuple(id) mlist = filter(lambda x: x.id in id, mlist) return list(mlist) if mtype: mtype = RockPy3._to_tuple(mtype) mtype = tuple(RockPy3.abbrev_to_classname(mt) for mt in mtype) mlist = filter(lambda x: x.mtype in mtype, mlist) if stype: mlist = filter(lambda x: x.has_stype(stype=stype, method='any'), mlist) if sval_range is not None: sval_range = self._convert_sval_range(sval_range=sval_range, mean=mean) if not sval: sval = sval_range else: sval = RockPy3._to_tuple() sval += RockPy3._to_tuple(sval_range) if sval is not None: mlist = filter(lambda x: x.has_sval(sval=sval, method='any'), mlist) if series: series = RockPy3.core.utils.tuple2list_of_tuples(series) mlist = (x for x in mlist if x.has_series(series=series, method='all')) if result: mlist = filter(lambda x: x.has_result(result=result), mlist) if invert: if mean: mlist = filter(lambda x: x not in mlist, self.mean_measurements) else: mlist = filter(lambda x: x not in mlist, self.measurements) return list(mlist)
def create_mean_measurement(self, mtype=None, stype=None, sval=None, sval_range=None, series=None, invert=False, mlist=None, interpolate=False, substfunc='mean', ignore_stypes=False, recalc_mag=False, reference=None, ref_dtype='mag', norm_dtypes='all', vval=None, norm_method='max', normalize_variable=False, dont_normalize=None, create_only=False, color=None, marker=None, linestyle=None): """ takes a list of measurements and creates a mean measurement out of all measurements data Parameters ---------- mlist: list list of measurements to be combined into a new measurement ignore_stypes: list stypes to be ignored. mtype: str mtype to be returned series: list(tuple) list of tuples, to search for several specific series. e.g. [('mtime',4),('gc',2)] will only return measurements that fulfill both criteria. Supersedes stype, sval and sval_range. Returns only measurements that meet series exactly! stype: str series type sval_range: list, str series range e.g. sval_range = [0,2] will give all from 0 to 2 including 0,2 also '<2', '<=2', '>2', and '>=2' are allowed. sval: float series value to be searched for. caution: will be overwritten when sval_range is given invert: if invert true it returns only measurements that do not meet above criteria sval_range: can be used to look up measurements within a certain range. if only one value is given, it is assumed to be an upper limit and the range is set to [0, sval_range] interpolate: bool substfunc: str recalc_mag: bool reference: str ref_dtype: str norm_dtypes: list vval: float norm_method: str normalize_variable: bool dont_normalize: list create_only: bool will not add measurement to the mean_measurements list or mean_mdict color: str color for new mean measurement marker: str marker for new mean measurement linestyle: str linestyle for new mean measurement Returns ------- RockPy.Measurement The mean measurement that fits to the specified lookup """ # check for mtype if no mtype specified if mlist and not mtype: mtype = list(set(m.mtype for m in mlist)) if len(mtype) != 1: raise TypeError('NO mtype specified. List of measurements may only contain one mtype') else: mtype = mtype[0] if not mtype and not mlist: raise TypeError('NO mtype specified. Please specify mtype') if not mlist: mlist = self.get_measurement(mtype=mtype, series=series, stype=stype, sval=sval, sval_range=sval_range, mean=False, invert=invert) # normalze all measurements if reference: mlist = [m.normalize(reference=reference, ref_dtype=ref_dtype, norm_dtypes=norm_dtypes, vval=vval, norm_method=norm_method, normalize_variable=normalize_variable, dont_normalize=dont_normalize) for m in mlist] if not mlist: self.log.warning('NO measurement found >> %s, %s, %f >>' % (mtype, stype, sval)) return None mtype = RockPy3.abbrev_to_classname(mtype) if len(mlist) == 1: self.log.warning('Only one measurement found returning measurement') # create mean measurement from a list of measurements mean = RockPy3.implemented_measurements[mtype].from_measurements_create_mean( sobj=self, mlist=mlist, interpolate=interpolate, recalc_mag=recalc_mag, substfunc=substfunc, ignore_stypes=ignore_stypes, color=color, marker=marker, linestyle=linestyle) mean.calc_all(force_recalc=True) # add to self.mean_measurements if specified if not create_only: self.mean_measurements.append(mean) return mean
def __init__(self, fpath, sgroups=None, samples=None, mtypes=None, ftype=None, mass=None, height=None, diameter=None, massunit=None, lengthunit=None, heightunit=None, diameterunit=None, series=None, comment=None, folder=None, suffix=None, read_fpath=True, **kwargs): """ Parameters ---------- fpath sgroups samples mtypes ftype mass height diameter massunit lengthunit heightunit diameterunit series comment folder suffix read_fpath: bool if true the path will be read for info kwargs """ if 'mtype' in kwargs and not mtypes: mtypes = kwargs.pop('mtype') if 'sgroup' in kwargs and not sgroups: mtypes = kwargs.pop('sgroup') if 'sample' in kwargs and not samples: mtypes = kwargs.pop('sample') blocks = (self.measurement_block, self.sample_block, self.series_block, self.add_block, self.comment_block) additional = tuple() sgroups = RockPy3._to_tuple(sgroups) sgroups = tuple([sg if sg != 'None' else None for sg in sgroups]) if mtypes: mtypes = tuple(RockPy3.abbrev_to_classname(mtype) for mtype in RockPy3._to_tuple(mtypes)) if ftype: ftype = RockPy3.abbrev_to_classname(ftype) self.__dict__.update({i: None for i in ('sgroups', 'samples', 'mtypes', 'ftype', 'mass', 'height', 'diameter', 'massunit', 'lengthunit', 'heightunit', 'diameterunit', 'series', 'additional', 'comment', 'folder', 'suffix') }) self.fpath = fpath if read_fpath and fpath: # todo add check for if path is readable self.folder = os.path.dirname(fpath) f, self.suffix = os.path.splitext(os.path.basename(fpath)) self.suffix = self.suffix.strip('.') splits = f.split('#') # check if RockPy compatible e.g. first part must be len(4) if not len(splits[0]) == 4: pass for i, block in enumerate(blocks[:len(splits)]): if splits[i]: try: block(splits[i]) except (ValueError,): pass for i in ('sgroups', 'samples', 'mtypes', 'ftype', 'mass', 'height', 'diameter', 'massunit', 'lengthunit', 'heightunit', 'diameterunit', 'series', 'additional', 'comment', 'folder'): if locals()[i]: if isinstance(locals()[i], (tuple, list, set)): if not all(locals()[i]): continue setattr(self, i, locals()[i]) if self.additional is None: self.additional = '' if kwargs: for k, v in kwargs.items(): if v: print(k, v, self.additional) self.additional += '{}:{}'.format(k, v) if suffix: self.suffix = suffix if type(self.suffix) == int: self.suffix = '%03i' % self.suffix if not self.suffix: self.suffix = '000' if not self.sgroups: self.sgroups = None self.storage = [[self.sgroups, self.samples, self.mtypes, self.ftype], [[self.mass, self.massunit], [self.height, self.heightunit], [self.diameter, self.diameterunit], ], self.series, (self.additional,), self.comment]
def __init__(self, fpath, sgroups=None, samples=None, mtypes=None, ftype=None, mass=None, height=None, diameter=None, massunit=None, lengthunit=None, heightunit=None, diameterunit=None, series=None, comment=None, folder=None, suffix=None, read_fpath=True, **kwargs): """ Parameters ---------- fpath sgroups samples mtypes ftype mass height diameter massunit lengthunit heightunit diameterunit series comment folder suffix read_fpath: bool if true the path will be read for info kwargs """ if 'mtype' in kwargs and not mtypes: mtypes = kwargs.pop('mtype') if 'sgroup' in kwargs and not sgroups: mtypes = kwargs.pop('sgroup') if 'sample' in kwargs and not samples: mtypes = kwargs.pop('sample') blocks = (self.measurement_block, self.sample_block, self.series_block, self.add_block, self.comment_block) additional = tuple() sgroups = RockPy3._to_tuple(sgroups) sgroups = tuple([sg if sg != 'None' else None for sg in sgroups]) if mtypes: mtypes = tuple( RockPy3.abbrev_to_classname(mtype) for mtype in RockPy3._to_tuple(mtypes)) if ftype: ftype = RockPy3.abbrev_to_classname(ftype) self.__dict__.update({ i: None for i in ('sgroups', 'samples', 'mtypes', 'ftype', 'mass', 'height', 'diameter', 'massunit', 'lengthunit', 'heightunit', 'diameterunit', 'series', 'additional', 'comment', 'folder', 'suffix') }) self.fpath = fpath if read_fpath and fpath: # todo add check for if path is readable self.folder = os.path.dirname(fpath) f, self.suffix = os.path.splitext(os.path.basename(fpath)) self.suffix = self.suffix.strip('.') splits = f.split('#') # check if RockPy compatible e.g. first part must be len(4) if not len(splits[0]) == 4: pass for i, block in enumerate(blocks[:len(splits)]): if splits[i]: try: block(splits[i]) except (ValueError, ): pass for i in ('sgroups', 'samples', 'mtypes', 'ftype', 'mass', 'height', 'diameter', 'massunit', 'lengthunit', 'heightunit', 'diameterunit', 'series', 'additional', 'comment', 'folder'): if locals()[i]: if isinstance(locals()[i], (tuple, list, set)): if not all(locals()[i]): continue setattr(self, i, locals()[i]) if self.additional is None: self.additional = '' if kwargs: for k, v in kwargs.items(): if v: print(k, v, self.additional) self.additional += '{}:{}'.format(k, v) if suffix: self.suffix = suffix if type(self.suffix) == int: self.suffix = '%03i' % self.suffix if not self.suffix: self.suffix = '000' if not self.sgroups: self.sgroups = None self.storage = [[self.sgroups, self.samples, self.mtypes, self.ftype], [ [self.mass, self.massunit], [self.height, self.heightunit], [self.diameter, self.diameterunit], ], self.series, (self.additional, ), self.comment]
def import_folder(self, folder, sname=None, sgroup=None, mtype=None, automatic_results = True, ): # todo specify samples, mtypes and series for selective import of folder """ imports all files in the specified folder Parameters ---------- folder gname Returns ------- """ files = [os.path.join(folder, i) for i in os.listdir(folder) if not i.startswith('#') if not i.startswith(".") if not i.endswith("pynb") if not i.endswith("log") if not os.path.isdir(os.path.join(folder, i)) ] measurements = [] start = time.clock() minfos = (RockPy3.core.file_operations.minfo(f) for f in files) minfos = (minfo for minfo in minfos if minfo.is_readable()) if mtype: mtype = RockPy3._to_tuple(mtype) mtype = [RockPy3.abbrev_to_classname(mt) for mt in mtype] for mi in minfos: RockPy3.logger.info('importing {}'.format(mi.fpath)) # dont import samples that are not in the measurement info if sname: sname = RockPy3._to_tuple(sname) if not any(s in mi.samples for s in sname): continue # dont import samples that are not included in a certain samplegroup in the measurement info if sgroup: sgroup = RockPy3._to_tuple(sgroup) if not any(sg in mi.sgroups for sg in sgroup): continue # check if the filename has not been imported, yet if mi.fpath in self.imported_files: continue else: self.imported_files.append(mi.fpath) for sinfo in mi.sample_infos: s = self.add_sample(warnings=False, **sinfo) for minfo in mi.measurement_infos: if mtype and minfo['mtype'].lower() not in mtype: continue if minfo['sample'] == sinfo['name']: measurements.append(s.add_measurement(**minfo)) end = time.clock() print('IMPORT generated {} measurements: finished in {:.02e}s'.format(len(measurements), end - start)) return measurements
def create_mean_measurement(self, mtype=None, stype=None, sval=None, sval_range=None, series=None, invert=False, mlist=None, interpolate=False, substfunc='mean', ignore_stypes=False, recalc_mag=False, reference=None, ref_dtype='mag', norm_dtypes='all', vval=None, norm_method='max', normalize_variable=False, dont_normalize=None, create_only=False, color=None, marker=None, linestyle=None): """ takes a list of measurements and creates a mean measurement out of all measurements data Parameters ---------- mlist: list list of measurements to be combined into a new measurement ignore_stypes: list stypes to be ignored. mtype: str mtype to be returned series: list(tuple) list of tuples, to search for several specific series. e.g. [('mtime',4),('gc',2)] will only return measurements that fulfill both criteria. Supersedes stype, sval and sval_range. Returns only measurements that meet series exactly! stype: str series type sval_range: list, str series range e.g. sval_range = [0,2] will give all from 0 to 2 including 0,2 also '<2', '<=2', '>2', and '>=2' are allowed. sval: float series value to be searched for. caution: will be overwritten when sval_range is given invert: if invert true it returns only measurements that do not meet above criteria sval_range: can be used to look up measurements within a certain range. if only one value is given, it is assumed to be an upper limit and the range is set to [0, sval_range] interpolate: bool substfunc: str recalc_mag: bool reference: str ref_dtype: str norm_dtypes: list vval: float norm_method: str normalize_variable: bool dont_normalize: list create_only: bool will not add measurement to the mean_measurements list or mean_mdict color: str color for new mean measurement marker: str marker for new mean measurement linestyle: str linestyle for new mean measurement Returns ------- RockPy.Measurement The mean measurement that fits to the specified lookup """ # check for mtype if no mtype specified if mlist and not mtype: mtype = list(set(m.mtype for m in mlist)) if len(mtype) != 1: raise TypeError( 'NO mtype specified. List of measurements may only contain one mtype' ) else: mtype = mtype[0] if not mtype and not mlist: raise TypeError('NO mtype specified. Please specify mtype') if not mlist: mlist = self.get_measurement(mtype=mtype, series=series, stype=stype, sval=sval, sval_range=sval_range, mean=False, invert=invert) # normalze all measurements if reference: mlist = [ m.normalize(reference=reference, ref_dtype=ref_dtype, norm_dtypes=norm_dtypes, vval=vval, norm_method=norm_method, normalize_variable=normalize_variable, dont_normalize=dont_normalize) for m in mlist ] if not mlist: self.log.warning('NO measurement found >> %s, %s, %f >>' % (mtype, stype, sval)) return None mtype = RockPy3.abbrev_to_classname(mtype) if len(mlist) == 1: self.log.warning( 'Only one measurement found returning measurement') # create mean measurement from a list of measurements mean = RockPy3.implemented_measurements[ mtype].from_measurements_create_mean(sobj=self, mlist=mlist, interpolate=interpolate, recalc_mag=recalc_mag, substfunc=substfunc, ignore_stypes=ignore_stypes, color=color, marker=marker, linestyle=linestyle) mean.calc_all(force_recalc=True) # add to self.mean_measurements if specified if not create_only: self.mean_measurements.append(mean) return mean
def get_measurement(self, mtype=None, series=None, stype=None, sval=None, sval_range=None, mean=False, invert=False, id=None, result=None): """ Returns a list of measurements of type = mtypes Parameters ---------- mtypes: list, str mtypes to be returned series: list(tuple) list of tuples, to search for several sepcific series. e.g. [('mtime',4),('gc',2)] will only return mesurements that fulfill both criteria. stypes: list, str series type sval_range: list, str series range e.g. sval_range = [0,2] will give all from 0 to 2 including 0,2 also '<2', '<=2', '>2', and '>=2' are allowed. svals: float series value to be searched for. caution: will be overwritten when sval_range is given invert: if invert true it returns only measurements that do not meet criteria sval_range: can be used to look up measurements within a certain range. if only one value is given, it is assumed to be an upper limit and the range is set to [0, sval_range] mean: bool id: list(int) search for given measurement id Returns ------- if no arguments are passed all sample.measurements list of RockPy.Measurements that meet search criteria or if invert is True, do not meet criteria. [] if none are found Note ---- there is no connection between stype and sval. This may cause problems. I you have measurements with M1: [pressure, 0.0, GPa], [temperature, 100.0, C] M2: [pressure, 1.0, GPa], [temperature, 100.0, C] and you search for stypes=['pressure','temperature'], svals=[0,100]. It will return both M1 and M2 because both M1 and M2 have [temperature, 100.0, C]. """ if mean: mlist = self.mean_measurements else: mlist = self.measurements if id is not None: id = RockPy3._to_tuple(id) mlist = filter(lambda x: x.id in id, mlist) return list(mlist) if mtype: mtype = RockPy3._to_tuple(mtype) mtype = tuple(RockPy3.abbrev_to_classname(mt) for mt in mtype) mlist = filter(lambda x: x.mtype in mtype, mlist) if stype: mlist = filter(lambda x: x.has_stype(stype=stype, method='any'), mlist) if sval_range is not None: sval_range = self._convert_sval_range(sval_range=sval_range, mean=mean) if not sval: sval = sval_range else: sval = RockPy3._to_tuple() sval += RockPy3._to_tuple(sval_range) if sval is not None: mlist = filter(lambda x: x.has_sval(sval=sval, method='any'), mlist) if series: series = RockPy3.core.utils.tuple2list_of_tuples(series) mlist = (x for x in mlist if x.has_series(series=series, method='all')) if result: mlist = filter(lambda x: x.has_result(result=result), mlist) if invert: if mean: mlist = filter(lambda x: x not in mlist, self.mean_measurements) else: mlist = filter(lambda x: x not in mlist, self.measurements) return list(mlist)