def find_one(path, jobname=None): """ Returns an executabe job-folder as a context. The goal is to retrieve an executable job-folder from disk, while using a lock-file to prevent other processes from reading/writing the job-folder at an importune time. The lock-file is not held throughout the existence of the context. It is only held for specific operations when entering and leaving the context. The function is a context_. As such, it is possible to modify the job-folder. The modifications will be saved when leaving the context. .. _context: http://docs.python.org/reference/datamodel.html#context-managers """ from ..misc import LockFile found = False # acquire lock. Yield context outside of lock! with LockFile(path) as lock: # Loads pickle. with open(self._filename, 'r') as file: jobfolder = pickle_load(file) # Finds first untagged job. if jobname is None: for job in jobfolder.itervalues(): if not job.is_tagged: found = True break else: job = jobfolder[jobname] # tag the job before surrendering the lock. if found and jobname is None: job.tag() with open(self._filename, 'w') as file: dump(jobfolder, file) # Check we found an untagged job. Otherwise, we are done. if not found: yield None return # context returns jobs. yield job # saves job since it might have been modified # acquire a lock first. with LockFile(self._filename) as lock: # Loads pickle. with open(self._filename, 'r') as file: jobfolder = pickle_load(file) # modifies job. jobfolder[job.name] = job # save jobs. with open(self._filename, 'w') as file: dump(jobfolder, file)
def an_092b_anpassen(self): if self.mb.debug: log(inspect.stack) # Tags_time in den Sidebar dict eintragen try: from pickle import load as pickle_load pfad = os.path.join(self.mb.pfade["files"], "sidebar_content.pkl") with open(pfad, "rb") as f: self.mb.dict_sb_content = pickle_load(f) dict_sb_content = self.mb.dict_sb_content dict = {} dict.update({"zeit": None}) dict.update({"datum": None}) for ordn in dict_sb_content["ordinal"]: if dict_sb_content["ordinal"][ordn]["Tags_time"] == "": self.mb.dict_sb_content["ordinal"][ordn]["Tags_time"] = dict self.mb.class_Sidebar.speicher_sidebar_dict() self.version = "0.9.2b" except: log(inspect.stack, tb())
def load_run(model, run_path): """ Load a saved model and (if existing) additional data from disk. :param model: the model to be updated with the saved state-dict :param run_path: the path to the folder containing the files to load :return: the model and, optionally, additional data """ if not run_path.is_dir(): raise NotADirectoryError(f"Invalid run-directory path.") model_files = [ file for file in run_path.iterdir() if file.suffix == ".pkt" ] data_files = [file for file in run_path.iterdir() if file.suffix == ".pkl"] if len(model_files) == 0: raise ValueError( f"There are no model files to load in {run_path.name}.") if len(model_files) > 1 or len(data_files) > 1: raise ValueError( f"There exists more than one model- and/or data file in {run_path.name}." ) model.load_state_dict(load(model_files[0])) print(f"Successfully loaded model data from {model_files[0].name}") if data_files: with open(data_files[0], 'rb') as data_file: data = pickle_load(data_file) print(f"Successfully loaded data from {data_files[0].name}") return model, data return model
def usePickleToMeta(): if azienda: meta_pickle = azienda + "-meta.pickle" + sys.version[:1] promogestDir = os.path.expanduser("~") + os.sep + "promogest2" + os.sep + azienda + os.sep meta_pickle_da_conf = main_conf.Database.azienda + "-meta.pickle" + sys.version[:1] promogestDir_da_conf = ( os.path.expanduser("~") + os.sep + "promogest2" + os.sep + main_conf.Database.azienda + os.sep ) else: meta_pickle = None meta_pickle_da_conf = None promogestDir_da_conf = None if ( tipo_eng != "sqlite" and azienda and os.path.exists(str(os.path.join(promogestDir.replace("_", ""), meta_pickle.replace("_", "")).strip())) ): print(" CONTROLLO DELL'ESISTENZA DEL FILE PICKLE", str(os.path.join(promogestDir.replace("_", "")))) with open(str(os.path.join(promogestDir.replace("_", ""), meta_pickle.replace("_", "")).strip()), "rb") as f: try: meta = pickle_load(f) meta.bind = engine except: print("DEVO CANCELLARE IL PICKLE PERCHé NON RIESCO A TROVARLO O LEGGERLO") delete_pickle() print("USO META PICKLE FAST") if not meta: meta = MetaData(engine) else: print("USO META NORMALE") meta = MetaData(engine) return meta
def init_session(remote_address, cookie_data=None, cookie=None): if cookie_data is not None: cookie = SessionCookie.load(cookie_data) # Default sid for the session sid = sha224(('%s-%s' % (remote_address, datetime.utcnow())).encode()).hexdigest() if cookie is None: cookie = SessionCookie(sid) else: try: cookie.get_sid() except KeyError: # For some reason the cookie did not contain a SID, set to default cookie.set_sid(sid) # Set the session singleton (there can be only one!) global CURRENT_SESSION ppath = get_session_pickle_path(cookie.get_sid()) if isfile(ppath): # Load our old session data and initialise the cookie try: with open(ppath, 'rb') as session_pickle: CURRENT_SESSION = pickle_load(session_pickle) CURRENT_SESSION.init_cookie(CURRENT_SESSION.get_sid()) except Exception as e: # On any error, just create a new session CURRENT_SESSION = Session(cookie) else: # Create a new session CURRENT_SESSION = Session(cookie)
def lade_Backup(self,fehlende = 'all'): if self.mb.debug: log(inspect.stack,None,'### Attention ###, sidebar_content.pkl.backup loaded!') pfad_Backup = os.path.join(self.mb.pfade['files'],'sidebar_content.pkl.Backup') if fehlende == 'all': fehlende = list(self.mb.props['Projekt'].dict_bereiche['ordinal']) try: with open(pfad_Backup, 'rb') as f: backup = pickle_load(f) except: log(inspect.stack,tb()) for f in fehlende: self.lege_dict_sb_content_ordinal_an(f) return helfer = fehlende[:] if backup != None: for f in fehlende: if f in backup['ordinal']: self.mb.dict_sb_content['ordinal'].update(backup['ordinal'][f]) helfer.remove(f) fehlende = helfer for f in fehlende: self.lege_dict_sb_content_ordinal_an(f)
def cache_calc(filename, func, *args, **kwargs): """ Cache calculations, so that the first call to this function performs the calculations, and caches them to a file. And future calls to this function simply load up the data from the cached file. :param filename:(str) the file path you want to save the cached file as :param func: The function to call to calculate the :param *args: ordered arguments to be passed on to func() :param **kwargs: keyword arguments to be passed on to func() :return: whatever func() returns. :examples: cache_calc("myCachedFile", myFunc) """ # ========================================================================== if file_exists(filename): print("Loading the cached version of " + filename) with open(filename, mode="rb") as fileObj: x = pickle_load(fileObj) else: print("Caching the calculation to the file " + filename) x = func(*args, **kwargs) # Cache the calculation so future calls to this function load the cached # object instead. with open(filename, mode="wb") as fileObj: pickle_dump(x, fileObj) return x
def load(self): fileName = config.plugins.birthdayreminder.file.value print("[Birthday Reminder] reading from file", fileName) tmpList = [] if isfile(fileName): try: f = open(fileName, "r") tmpList = pickle_load(f) f.close() except IOError as xxx_todo_changeme1: (error_no, error_str) = xxx_todo_changeme1.args print( "[Birthday Reminder] ERROR reading from file %s. Error: %s, %s" % (fileName, error_no, error_str)) text = _("Error reading file %s.\n\nError: %s, %s") % ( fileName, error_no, error_str) Notifications.AddNotification(MessageBox, text, type=MessageBox.TYPE_ERROR) print("[Birthday Reminder] read %s birthdays" % len(tmpList)) else: print("[Birthday Reminder] File %s not found." % fileName) self.bDayList = tmpList
def lade_sidebar_dict(self): if self.mb.debug: log(inspect.stack) pfad = os.path.join(self.mb.pfade['files'],'sidebar_content.pkl') dict_exists, backup_exists = False, False if os.path.exists(pfad): dict_exists = True if not os.path.exists(pfad+'.Backup'): backup_exists = False try: if dict_exists: with open(pfad, 'rb') as f: self.mb.dict_sb_content = pickle_load(f) self.ueberpruefe_dict_sb_content(not backup_exists) elif not backup_exists: self.lege_dict_sb_content_an() self.lade_Backup() except: log(inspect.stack,tb()) self.lege_dict_sb_content_an() if not backup_exists: self.lade_Backup() if not dict_exists: self.speicher_sidebar_dict() self.erzeuge_dict_sb_content_Backup()
def _get(cache_key, file_name, override_expire_secs=None, pickle=False): expire_datetime = None cache_path = xbmc_helper().get_file_path(CONST['CACHE_DIR'], file_name) if (override_expire_secs is not None): expire_datetime = datetime.now() - timedelta(seconds=override_expire_secs) elif 'expires' in CONST['CACHE'][cache_key].keys() and CONST['CACHE'][cache_key]['expires'] is not None: expire_datetime = datetime.now() - timedelta(seconds=CONST['CACHE'][cache_key]['expires']) cache_data = { 'data': None, 'is_expired': True, } if path.exists(cache_path): filectime = datetime.fromtimestamp(path.getctime(cache_path)) filemtime = datetime.fromtimestamp(path.getmtime(cache_path)) if filemtime is None or filectime > filemtime: filemtime = filectime if pickle is False: with io_open(file=cache_path, mode='r', encoding='utf-8') as cache_infile: cache_data.update({'data': cache_infile.read()}) else: with io_open(file=cache_path, mode='rb') as cache_infile: cache_data.update({'data': pickle_load(cache_infile)}) if expire_datetime is None or filemtime >= expire_datetime: cache_data.update({'is_expired': False}) return cache_data
def __enter__( self ): if self._mode == "r": with open( self._fn, "rb" ) as f: state = pickle_load( f ); self._len_c = state[ "c" ]; self._len_b = state[ "b" ]; self._len_x = state[ "x" ]; self._lenrow = self._len_c + self._len_b + self._len_x; self._ic = state[ "ic" ]; self._icbp = state[ "icbp" ]; if self._mode == "w": with NamedTemporaryFile() as tmpfn: self._kdbfn = tmpfn.name + '.kch'; self._kdb = KDB(); try: assert self._kdb.open( self._kdbfn, KDB.OWRITER | KDB.OCREATE ); except: print( str( self._kdb.error() ) ); raise; with TemporaryDirectory() as tmpdirname: self._ldbdn = tmpdirname; self._ldb = LDB( self._ldbdn, create_if_missing=True ); return self;
def init_session(remote_address, cookie_data=None): if cookie_data is not None: cookie = SessionCookie.load(cookie_data) else: cookie = None # Default sid for the session sid = sha224('%s-%s'.format( remote_address, datetime.utcnow()).encode('utf-8')).hexdigest() if cookie is None: cookie = SessionCookie(sid) else: try: cookie.get_sid() except KeyError: # For some reason the cookie did not contain a SID, set to default cookie.set_sid(sid) # Set the session singleton (there can be only one!) global CURRENT_SESSION ppath = get_session_pickle_path(cookie.get_sid()) if isfile(ppath): # Load our old session data and initialise the cookie try: with open(ppath, 'rb') as session_pickle: CURRENT_SESSION = pickle_load(session_pickle) CURRENT_SESSION.init_cookie(CURRENT_SESSION.get_sid()) except Exception as e: # On any error, just create a new session CURRENT_SESSION = Session(cookie) else: # Create a new session CURRENT_SESSION = Session(cookie)
def random_walk(seed_node, n, folNm): # in graph G neg_comp = nx_Graph() neg_comp.add_node(seed_node) node_num = 1 pres_node = seed_node extra = 10 while node_num < n + extra: with open(folNm + "/" + pres_node, 'rb') as f: neig_list = pickle_load(f) if not neig_list: logging_debug("No neighbours") break if len(neig_list) != 1: new_node = rand_choice(list(neig_list.keys())) else: new_node = list(neig_list.keys())[0] wt = neig_list[new_node] wt_edge = wt['weight'] neg_comp.add_edge(pres_node, new_node, weight=wt_edge) if len(neg_comp.nodes()) == n: break pres_node = new_node node_num = node_num + 1 # print(pres_node) return neg_comp
def loaddump(filename, silent=False): """Unserializes data from a file that was written with the built-in cPickle module. .. Seealso:: :func:`savedump` """ assert filename is not None and filename != "" and filename != "-", filename if not exists(filename): if not silent: print("loaddump(): File '" + filename + "' does not exist. Returning None.") return None try: if not silent: print("loaddump(): Loading data from '" + filename + "'..."), with open(filename, "rb") as f: data = pickle_load(f) if not silent: print("Done") return data except Exception as ex: print("ERROR: loaddump(): Could not load data from '" + filename + "'.") print(" Passing exception to caller.") print(str(ex)) raise ex
def getRelationDict(): if os_path.exists(CONFIG_FILE): pkl_file = open(CONFIG_FILE, 'rb') if pkl_file: volumedict = pickle_load(pkl_file) pkl_file.close() return volumedict return {}
def search_metropolis(seed_node, scaler, par_inputs_fn): # Picks out of a subset of its neighbors and adds the best node with open(par_inputs_fn, 'rb') as f: inputs = pickle_load(f) with open(inputs['modelfname'], 'rb') as f: model = pickle_load(f) folNm = inputs['folNm'] folNm_out = inputs['folNm_out'] cd, g1 = starting_edge(folNm, seed_node) if cd == 0: return a, b = met(g1, model, scaler, inputs, 0) with open(folNm_out + "/" + seed_node, 'wb') as f: pickle_dump((a, b), f)
def file_pickle_load(path: str): if file_test(path): with open(path, "rb") as f: t = pickle_load(f) return t print(f"Pickle-File {path} could not be found!") return None
def pickle_restore(pickle_file_path): '''Load data from pickle file''' try: with open(pickle_file_path, "rb") as f: last_session_data = pickle_load(f) except Exception: return None return last_session_data
def getVolumeDict(): if os_path.exists(CONFIG_FILE_VOLUME): pkl_file = open(CONFIG_FILE_VOLUME, 'rb') if pkl_file: volumedict = pickle_load(pkl_file) pkl_file.close() return volumedict return {}
def __enter__( self ): self._cdata = DB(); self._ddata = DB(); try: if self._mode == "r": assert self._cdata.open( self._fn_cdata, DB.OREADER ); elif self._mode == "w": if isfile( self._fn_cdata ): remove( self._fn_cdata ); assert self._cdata.open( self._fn_cdata, DB.OWRITER | DB.OCREATE ); else: assert False; except: if self._cdata is not None: print( str( self._cdata.error() ) ); raise; try: if self._mode == "r": assert self._ddata.open( self._fn_ddata, DB.OREADER ); elif self._mode == "w": if isfile( self._fn_ddata ): remove( self._fn_ddata ); assert self._ddata.open( self._fn_ddata, DB.OWRITER | DB.OCREATE ); else: assert False; except: if self._ddata is not None: print( str( self._ddata.error() ) ); raise; if self._mode == "r": with open( self._fn_meta, 'rb' ) as f: r = pickle_load( f ); self._len_c = r[ "c" ]; self._len_b = r[ "b" ]; self._len_x = r[ "x" ]; self._co = r[ "co" ]; with open( self._fn_icov, 'rb' ) as f: self._icov = pickle_load( f ); return self;
def fill_json(apps, schema_editor): for lwemail in apps.get_model('emails', 'LightWeightEmail').objects.all(): body = lwemail.body if body: # lwemail.body = b'' lwemail.body_tmp = json_dump( pickle_load(body.encode(), encoding='utf-8')) lwemail.save()
def search_metropolis_clique_start(scaler,par_inputs_fn,G_clique): # Picks out of a subset of its neighbors and adds the best node #print(seed_clique) with open(par_inputs_fn,'rb') as f: inputs = pickle_load(f) with open(inputs['modelfname'],'rb') as f: model = pickle_load(f) g1=nx_Graph(G_clique) # Finding score (score_prev,comp_bool) = get_score(g1,model,scaler,inputs['model_type']) # Removing starting points which are not complexes if comp_bool == 0: return ([],0) g1 = met(g1,model,scaler,inputs,score_prev) return g1
def load(self, path): with open(path, 'rb') as f: try: odata = pickle_load(f) self.data = odata['data'] self._mc = odata['mc'] self.classifier = odata['classifier'] except Exception as e: print("Invalid data: %s" % (str(e)))
def _get(self): """ Retrieves the data set model :return: map, [instrument][composer][piece][tempo] = abs_download_path """ mapping = {} urls = [] website_dump = OneClassicalDataSet.__name__ + ".dump.pkl" if exists("dump.pkl"): pp("Restoring Website Information...") urls, mapping = pickle_load(open(website_dump, "rb")) else: pp("Retrieving Website Information for " + self.url) for ref_instrument in self._get_ref_links(self.url + self.by_instrument): instrument = ref_instrument.split("=")[1] # print ("Instrument: ", ref_instrument, instrument) mapping[instrument] = {} for ref_composer in self._get_ref_links(self.url + ref_instrument): composer = ref_composer.split("=")[1].split("-")[1].strip() # print ("Composer: ", ref_composer, composer) mapping[instrument][composer] = {} for ref_piece in self._get_ref_links(self.url + ref_composer): piece = ref_piece.split("=")[1].split("-")[1].strip() downloads = self._get_downloads(self.url + ref_piece) print(instrument, composer, piece, downloads) mapping[instrument][composer][piece] = downloads urls += list( map(lambda tempo: downloads[tempo], downloads)) pickle_dump((urls, mapping), open(website_dump, "wb")) # Download Data Set content = self._download(urls=list(set(urls))) # Mapping File Paths to Data Set to_remove = [] for i in mapping: for c in mapping[i]: for p in mapping[i][c]: for t in mapping[i][c][p]: if mapping[i][c][p][t] in content: mapping[i][c][p][t] = content[mapping[i][c][p][t]] else: to_remove.append([i, c, p, t]) # Remove all mapping where audio could not be downloaded for item in to_remove: del (mapping[item[0]][item[1]][item[2]][item[3]]) # Remove website dump remove(website_dump) return mapping
def get_resource(self, name: str) -> Any: """Get the resource registered under a given name.""" if name not in self._instance_registry: raise ResourceException(f"Resource {name} not found.") if self._cache_dir: cache_path = self._name_cache_path(name) if cache_path.is_file(): with bz2_open(cache_path, "rb") as fh: return pickle_load(fh) return self._instance_registry[name]
def search_metropolis_clique_start(scaler, par_inputs_fn, G_clique): # Picks out of a subset of its neighbors and adds the best node # print(seed_clique) with open(par_inputs_fn, 'rb') as f: inputs = pickle_load(f) with open(inputs['modelfname'], 'rb') as f: model = pickle_load(f) g1 = nx_Graph(G_clique) # Finding score score_prev, comp_bool = get_score(g1, model, scaler, inputs['model_type']) # Removing starting points which are not complexes if score_prev < inputs["classi_thresh"]: return a, b = met(g1, model, scaler, inputs, score_prev) name = " ".join([str(n) for n in g1.nodes()]) with open(folNm_out + "/" + name, 'wb') as f: pickle_dump((a, b), f)
def load_pickle(name, raise_exception=False): location = join(conf.DIRECTORY, 'pickles', '{}.pickle'.format(name)) try: with open(location, 'rb') as f: return pickle_load(f) except (FileNotFoundError, EOFError): if raise_exception: raise FileNotFoundError else: return None
def load(filename): """Load a template from ``filename``, return ``Templater`` object. This method must be used in pair with ``Templater.dump`` - it loads the template definition from a file using cPickle, creates a ``Templater`` object with the definition and returns it. """ with open(filename, 'rb') as loaded_file: processed_template = pickle_load(loaded_file) return processed_template
def _load(self): if not self._cache_filename: return {} try: with open(self._cache_filename) as f: return pickle_load(f) except IOError: #print('Cannot load cache from file {0}' # .format(self._cache_filename)) return {}
def __init__(self, conn_string: str) -> None: super().__init__(conn_string) try: with open(self.conn_string, 'rb') as f: self.values = pickle_load(f) except OSError as e: self.logger.warning( 'Can\'t open file "%s": "%s". Cache unavailable.', self.conn_string, e) self.values = {}
def _load_pion_function(self): """Returns the universal function on a fixed energy range """ from pickle import load as pickle_load from scipy.interpolate import UnivariateSpline uf_file = join(config.data_dir, 'pion_spline.pkl') with open(uf_file, 'rb') as f: tck = pickle_load(f, encoding='latin1') self.pion_spl = UnivariateSpline._from_tck(tck)
def _init_mainloop(self) -> None: try: with open(self._SERVER_CACHE_FILE, "rb") as cache_file, self._cache_lock: self._dns_cache = pickle_load(cache_file) self._expiration_dict = pickle_load(cache_file) self._last_check_time = pickle_load(cache_file) os_remove(self._SERVER_CACHE_FILE) except OSError: self._dns_cache = dict() self._expiration_dict = dict() self._last_check_time = int(time_time()) console_listener = Thread(target=self._console_listening, daemon=True) console_listener.start() expiration_checker = Thread(target=self._expiration_checking, daemon=True) expiration_checker.start()
def _load_pion_function(self): """Returns the universal function on a fixed energy range """ from pickle import load as pickle_load from scipy.interpolate import UnivariateSpline uf_file = join(global_path, 'data/pion_spline.pkl') with open(uf_file, 'r') as f: tck = pickle_load(f) self.pion_spl = UnivariateSpline._from_tck(tck)
def search_max_neig(seed_node, scaler, par_inputs_fn): with open(par_inputs_fn, 'rb') as f: inputs = pickle_load(f) with open(inputs['modelfname'], 'rb') as f: model = pickle_load(f) # Seed node logging_debug("Seed node is", seed_node) folNm = inputs['folNm'] folNm_out = inputs['folNm_out'] max_nodes = inputs["max_size"] score_curr = 0 cd, g1 = starting_edge(folNm, seed_node) if cd == 0: return while len(g1) < max_nodes: # print(len(g1)) logging_debug("Adding next node") neig_list = read_neigs(g1.nodes(), folNm) if not neig_list: # Checking if empty logging_debug("No more neighbors to add") break node_to_add = max(neig_list.items(), key=lambda elem: elem[1]['weight'])[0] g1 = add_newnode(g1, node_to_add, neig_list[node_to_add]['graph_neigs']) score_prev = score_curr (score_curr, comp_bool) = get_score(g1, model, scaler, inputs['model_type']) if score_curr < inputs["classi_thresh"]: logging_debug("Complex found") # Remove the node last added g1.remove_node(node_to_add) score_curr = score_prev break with open(folNm_out + "/" + seed_node, 'wb') as f: pickle_dump((frozenset(g1.nodes()), score_curr), f)
def register_resource(self, name: str, resource: Any) -> None: if name in self._instance_registry: raise ResourceException(f"Resource name {name} already in use.") if self._cache_dir: cache_path = self._name_cache_path(name) if cache_path.is_file(): self._logger.debug( f"Using the cached version of the {name} resource") with bz2_open(cache_path, "rb") as fh: resource = pickle_load(fh) self._instance_registry[name] = resource self._name_registry[id(resource)] = name
def unpickle_info(filename): """ @param filename: filename @type filename: str @return: unpickled info @rtype: T """ from pickle import load as pickle_load with open(filename, 'rb') as f: info = pickle_load(f) return info
def __init__(self, login, password, cookies=None): self._login = login self._password = password self._session = Session() self._cookies = cookies if cookies is not None: try: with open(cookies) as f: self._session.cookies = cookiejar_from_dict(pickle_load(f)) except IOError: _logger.error('Could not load cookies from {0}'.format(self._cookies))
def load_result(filename): """ Load (un-pickle) a Result object (or any other object) """ # if npz: # npd = numpy_load(filename) # result = Result(Response(npd['input_matrix'], npd['corr_names'], npd['mon_names'], list(npd['line']), # 'd_jw' in result, assume_sorted=True)) # else: with open(filename, 'rb') as f: result = pickle_load(f) return result
def get_prot_list(test_complex_path): with open(test_complex_path, 'rb') as f: test_complex_list = pickle_load(f) test_complex_nodes = [ item for sublist in test_complex_list for item in sublist ] test_prot_list = set(test_complex_nodes) with open(test_complex_path + "_prot_list", 'wb') as f: pickle_dump(test_prot_list, f) return test_prot_list
def init_data(): global data try: db_file = open("backend/db.p", "rb") data = pickle_load(db_file) db_file.close() print('load success!') print(data) except FileNotFoundError: reset_data() print('database reset!') Thread(target=commit_data_timer).start()
def load(path): """Load variable from Pickle file Args: path (str): path of the file to load Returns: variable read from path """ with open(path, 'rb') as f: variable = pickle_load(f) return variable
def step15( datadir, subsample ): data_pos = []; data_neg = []; for i in range( 0, len(DIMCLUSTERs) ): data_pos.append( [] ); data_neg.append( [] ); print( data_pos, data_neg ); with open( datadir+'/step07_'+subsample+'.pickle', 'rb' ) as f: data = pickle_load( f ); rowcount = 0; for ( y, x ) in data: rowcount += 1; if rowcount > 10000: break; for ( i, dims ) in enumerate( DIMCLUSTERs ): xval = 0.0; nx = 0.0; for dim in dims: if dim < 0: xval += -x[ abs(dim)-1 ]; else: xval += x[ abs(dim)-1 ]; nx += 1.0; if y == '0': data_neg[ i ].append( xval/nx ); elif y == '1': data_pos[ i ].append( xval/nx ); data_pos = np.array( data_pos ); data_neg = np.array( data_neg ); print( np.cov( data_neg ) ); n = len(DIMCLUSTERs); ( fig, ax ) = plt.subplots( nrows = n, ncols = n, figsize = ( 3*n, 3*n ) ); for i in range( 0, n ): for j in range( 0, n ): if j >= i: continue; ax[i,j].plot( data_neg[i], data_neg[j], marker='o', color='b', linestyle='', alpha=0.66 ); ax[i,j].plot( data_pos[i], data_pos[j], marker='o', color='r', linestyle='', alpha=0.66 ); ax[i,j].set_title( str((i,j)) ); fig.savefig( datadir+'/step15.png' );
def setUpClass(cls): """ Setup! """ here = dirname(__file__).replace('/', '\\') here = str(here) cls.raw_logger_settings_input = here + "\\input\\input_raw_logger_settings.cfg" cls.raw_logger_settings_output = here + "\\output\\output_raw_logger_settings.pickle" cls.raw_to_pynames_output = here + '\\output\\output_raw_to_pynames.pickle' cls.pynames_to_rvars_output = here + '\\output\\output_pyname_to_recipevar.pickle' with open(cls.raw_logger_settings_output, 'rb') as f: cls.exp_raw_logger_vars = pickle_load(f) cls.raw_to_pynames_input = cls.exp_raw_logger_vars with open(cls.raw_to_pynames_output, 'rb') as f: cls.exp_raw_to_pynames = pickle_load(f) cls.pynames_to_rvars_input = cls.exp_raw_to_pynames with open(cls.pynames_to_rvars_output, 'rb') as f: cls.exp_rvar_dict = pickle_load(f)
def load(self): fileName = config.plugins.birthdayreminder.file.value print "[Birthday Reminder] reading from file", fileName tmpList = [] if isfile(fileName): try: f = open(fileName, "r") tmpList = pickle_load(f) f.close() except IOError, (error_no, error_str): print "[Birthday Reminder] ERROR reading from file %s. Error: %s, %s" % (fileName, error_no, error_str) text = _("Error reading file %s.\n\nError: %s, %s") % (fileName, error_no, error_str) Notifications.AddNotification(MessageBox, text, type = MessageBox.TYPE_ERROR) print "[Birthday Reminder] read %s birthdays" % len(tmpList)
def __init__(self, folder, file, reset=False, lazy_update=False): self._file = os_path_join(folder, file) self._cache = cache = {} self._last = None if reset: return # If cache file already exists try: with open(self._file, 'rb') as file: for filepath, checksum in pickle_load(file).items(): # If file still exists if os_path_isfile(filepath): cache[filepath] = checksum if lazy_update: self._lcache = cache.copy() except (FileNotFoundError, EOFError): pass
def stop_stats(uri): from pickle import load as pickle_load from os.path import join as path_join cb = get_gfs2_callbacks() opq = cb.volumeStartOperations(uri, 'w') stats_obj = path_join( "/var/run/sr-private", cb.getUniqueIdentifier(opq), 'stats.obj' ) proc = None with open(stats_obj, 'r') as f: proc = pickle_load(f) proc.kill() force_unlink(stats_obj)
def usePickleToMeta(): if azienda and os.path.exists(str(os.path.join(promogestDir.replace("_",""),meta_pickle.replace("_","")).strip())): print " CONTROLLO DELL'ESISTENZA DEL FILE PICKLE", str(os.path.join(promogestDir.replace("_",""))) with open(str(os.path.join(promogestDir.replace("_",""),meta_pickle.replace("_","")).strip()), 'rb') as f: try: meta = pickle_load(f) meta.bind = engine #try: #meta.tables[azienda+".articolo"] #except: #delete_pickle() except: print "DEVO CANCELLARE IL PICKLE PERCHé NON RIESCO A TROVARLO O LEGGERLO" delete_pickle() print "USO META PICKLE FAST" #meta = MetaData(engine) else: print "USO META NORMALE" meta = MetaData(engine) return meta
def step09( datadir ): with open( datadir+'/step07_data.pickle', 'rb' ) as f: data = pickle_load( f ); neg_x1 = {}; neg_x2 = {}; pos_x1 = {}; pos_x2 = {}; for (y,x) in data: for (dim1,dim2) in INTERESTING_COMBINATIONs: dimpair = "{}_{}".format( dim1, dim2 ); x1 = x[ dim1 ]; x2 = x[ dim2 ]; if y == '0': neg_x1[ dimpair ] = neg_x1.get( dimpair, [] ) + [ x1 ]; neg_x2[ dimpair ] = neg_x2.get( dimpair, [] ) + [ x2 ]; elif y == '1': pos_x1[ dimpair ] = pos_x1.get( dimpair, [] ) + [ x1 ]; pos_x2[ dimpair ] = pos_x2.get( dimpair, [] ) + [ x2 ]; else: assert False; clusters1 = set(); clusters2 = set(); clusters3 = set(); for (dim1,dim2) in INTERESTING_COMBINATIONs: dimpair = "{}_{}".format( dim1, dim2 ); ( fig, ax ) = plt.subplots( nrows=1, ncols=1, figsize=(6,6) ); ax.plot( neg_x1[ dimpair ], neg_x2[ dimpair ], marker='o', color='b', linestyle='', alpha=0.66 ); ax.plot( pos_x1[ dimpair ], pos_x2[ dimpair ], marker='o', color='r', linestyle='', alpha=0.66 ); fig.savefig( datadir+'/step09.png' );
def step17( datadir, subsample ): data_pos = []; data_neg = []; with open( datadir+'/step07_'+subsample+'.pickle', 'rb' ) as f: data = pickle_load( f ); rowcount = 0; for ( y, x ) in data: rowcount += 1; if rowcount > 2500: break; row = []; for dim in DIMs: row.append( x[ abs(dim)-1 ] ); if y == '0': data_neg.append( row ); elif y == '1': data_pos.append( row ); print( len(data_pos), len(data) ); ratio = float( len(data_pos) ) / float( len(data) ); pthresholds \ = [ ratio * ( float(i) / 10.0 ) for i in range(5,26) ]; dthresholds \ = [ ( 0.5 + 0.1417 * float(len(DIMs)-3) ) * float(i)/10.0 \ for i in range(5,16) ]; print( ", ".join( [ "{:1.4f}".format(p) for p in pthresholds ] ) ); print( ", ".join( [ "{:1.4f}".format(d) for d in dthresholds ] ) ); cov = np.cov( np.array( data_pos+data_neg ).T ); cov_inv = LA.inv( cov ); print( cov ); print( cov_inv ); data_pos = np.array( data_pos ); data_neg = np.array( data_neg ); with open( datadir+'/step17_'+subsample+'.csv', 'wt' ) as out: for p_threshold in pthresholds: for d_threshold in dthresholds: print( ' {:1.2f} {:1.2f}'.format( p_threshold, d_threshold ) ); total_covered = set(); pos_covered = set(); for (i,ref_row) in enumerate( data_pos ): pos_in_vicinity = set(); neg_in_vicinity = set(); ref_row_vec = ref_row.reshape(1,len(ref_row)).T; for (j,pos_row) in enumerate( data_pos ): pos_row_vec = pos_row.reshape(1,len(pos_row)).T; diff = pos_row_vec - ref_row_vec; if USE_MAHALANOBIS: dist = np.sqrt( np.dot( np.dot( diff.T, cov_inv ), diff ) ); else: dist = LA.norm( diff ); if dist <= d_threshold: pos_in_vicinity.add( j ); for (j,neg_row) in enumerate( data_neg ): neg_row_vec = neg_row.reshape(1,len(neg_row)).T; diff = neg_row_vec - ref_row_vec; if USE_MAHALANOBIS: dist = np.sqrt( np.dot( np.dot( diff.T, cov_inv ), diff ) ); else: dist = LA.norm( diff ); if dist <= d_threshold: neg_in_vicinity.add( j ); total_in_vicinity = len( pos_in_vicinity ) + len( neg_in_vicinity ); if total_in_vicinity < 7: continue; p = float( len( pos_in_vicinity ) ) / float( total_in_vicinity ); if p > p_threshold: total_covered |= pos_in_vicinity; total_covered |= neg_in_vicinity; pos_covered |= pos_in_vicinity; # print( ' ', d_threshold, *stats ); if ( len(total_covered) > 1 ): recall = float( len(pos_covered) ) / float( len(data_pos) ); precision = float( len(pos_covered) ) / float( len(total_covered) ); f1 = 2.0 * ( precision * recall ) / ( precision + recall ); print( '-> {:1.2f} {:1.2f} {:1.4f} {:1.4f} {:1.4f}'.format( p_threshold, d_threshold, f1, precision, recall ) ); print( '{:1.2f};{:1.2f};{:1.4f};{:1.4f};{:1.4f}'.format( p_threshold, d_threshold, f1, precision, recall ), file=out );
def __enter__( self ): if self._mode == "r": with open( self._fn, "rb" ) as f: self._state = pickle_load( f ); return self;
def step08( datadir, subsample ): with open( datadir+'/step07_'+subsample+'.pickle', 'rb' ) as f: data = pickle_load( f ); values_by_dim = {}; values_by_dim_pos = {}; values_by_dim_neg = {}; for ( y, x ) in data: for ( i, x_val ) in enumerate( x ): values_by_dim[ i ] = values_by_dim.get( i, [] ) + [ x_val ]; if y == '0': values_by_dim_neg[ i ] = values_by_dim_neg.get( i, [] ) + [ x_val ]; elif y == '1': values_by_dim_pos[ i ] = values_by_dim_pos.get( i, [] ) + [ x_val ]; else: assert False; septiles_by_dim = {}; septiles_by_dim_pos = {}; septiles_by_dim_neg = {}; for i in values_by_dim: values_by_dim[ i ].sort(); values_by_dim_pos[ i ].sort(); values_by_dim_neg[ i ].sort(); N = float( len( values_by_dim[ i ] ) ); Np = float( len( values_by_dim_pos[ i ] ) ); Nn = float( len( values_by_dim_neg[ i ] ) ); septiles_by_dim[ i ] \ = ( values_by_dim[ i ][ int( (0.0*N)/7.0 ) ], values_by_dim[ i ][ int( (1.0*N)/7.0 ) ], values_by_dim[ i ][ int( (2.0*N)/7.0 ) ], values_by_dim[ i ][ int( (3.0*N)/7.0 ) ], values_by_dim[ i ][ int( (4.0*N)/7.0 ) -1 ], values_by_dim[ i ][ int( (5.0*N)/7.0 ) -1 ], values_by_dim[ i ][ int( (6.0*N)/7.0 ) -1 ], values_by_dim[ i ][ int( (7.0*N)/7.0 ) -1 ] ); septiles_by_dim_pos[ i ] \ = ( values_by_dim_pos[ i ][ int( (0.0*Np)/7.0 ) ], values_by_dim_pos[ i ][ int( (1.0*Np)/7.0 ) ], values_by_dim_pos[ i ][ int( (2.0*Np)/7.0 ) ], values_by_dim_pos[ i ][ int( (3.0*Np)/7.0 ) ], values_by_dim_pos[ i ][ int( (4.0*Np)/7.0 ) -1 ], values_by_dim_pos[ i ][ int( (5.0*Np)/7.0 ) -1 ], values_by_dim_pos[ i ][ int( (6.0*Np)/7.0 ) -1 ], values_by_dim_pos[ i ][ int( (7.0*Np)/7.0 ) -1 ] ); septiles_by_dim_neg[ i ] \ = ( values_by_dim_neg[ i ][ int( (0.0*Nn)/7.0 ) ], values_by_dim_neg[ i ][ int( (1.0*Nn)/7.0 ) ], values_by_dim_neg[ i ][ int( (2.0*Nn)/7.0 ) ], values_by_dim_neg[ i ][ int( (3.0*Nn)/7.0 ) ], values_by_dim_neg[ i ][ int( (4.0*Nn)/7.0 ) -1 ], values_by_dim_neg[ i ][ int( (5.0*Nn)/7.0 ) -1 ], values_by_dim_neg[ i ][ int( (6.0*Nn)/7.0 ) -1 ], values_by_dim_neg[ i ][ int( (7.0*Nn)/7.0 ) -1 ] ); stats_by_dimpair = {}; for ( y, x ) in data: for dim1 in septiles_by_dim: x1 = discretize( x[dim1], septiles_by_dim[dim1] ); for dim2 in septiles_by_dim: if dim1 >= dim2: continue; x2 = discretize( x[dim2], septiles_by_dim[dim2] ); if (dim1,dim2) not in stats_by_dimpair: stats_by_dimpair[ (dim1,dim2) ] = {}; ( total, pos ) = stats_by_dimpair[ (dim1,dim2) ].get( (x1,x2), (0,0) ); total += 1; if y == '1': pos += 1; stats_by_dimpair[ (dim1,dim2) ][ (x1,x2) ] = ( total, pos ); maxprop_dimpair = []; for ( dimpair, stats_by_valuepair ) in stats_by_dimpair.items(): maxprop = None; maxtotal = 0; for ( total, pos ) in stats_by_valuepair.values(): if total < 10: continue; prop = float(pos) / float(total); if maxprop is None: maxprop = prop; else: if prop > maxprop: maxprop = prop; maxtotal = total; maxprop_dimpair.append( (-maxprop,dimpair,maxtotal) ); with open( datadir+"/step08_"+subsample+"_median_shift_by_dim.txt", "wt" ) as out: median_shift_dim = []; for i in values_by_dim: median_pos = ( septiles_by_dim_pos[ i ][ 3 ] + septiles_by_dim_pos[ i ][ 4 ] ) / 2.0; median_neg = ( septiles_by_dim_neg[ i ][ 3 ] + septiles_by_dim_neg[ i ][ 4 ] ) / 2.0; median_shift_dim.append( ( abs(median_pos - median_neg), i, median_pos - median_neg ) ); for ( abs_median_shift, dim, median_shift ) in sorted( median_shift_dim, reverse=True ): print( dim, median_shift, sep=';' ); print( dim, median_shift, sep=';', file=out ); with open( datadir+"/step08_"+subsample+"_interesting_combinations.txt", "wt" ) as out: for (maxprop,dimpair,maxtotal) in sorted( maxprop_dimpair ): print( dimpair[0], dimpair[1], -maxprop, maxtotal, sep=';' ); print( dimpair[0], dimpair[1], -maxprop, maxtotal, sep=';', file=out );
def lade_tags(self): if self.mb.debug: log(inspect.stack) try: pfad = os.path.join(self.mb.pfade['settings'],'tags.pkl') with open(pfad, 'rb') as f: self.mb.tags = pickle_load(f) self.mb.class_Sidebar.offen = {nr:1 for nr in self.mb.tags['abfolge']} except: log(inspect.stack,tb()) p = Popup(self.mb, zeit=3) p.text = LANG.TAGS_NICHT_GEFUNDEN self.lege_tags_an() self.mb.class_Sidebar.offen = {nr:1 for nr in self.mb.tags['abfolge']} # Tags ueberpruefen try: ordinale = list(self.mb.props['ORGANON'].dict_bereiche['ordinal']) ordinale_tags = list(self.mb.tags['ordinale']) o_nicht_in_tags = [o for o in ordinale if o not in ordinale_tags] ueberfluessige_tags = [o for o in ordinale_tags if o not in ordinale] # 1) fehlende Tags neu anlegen if o_nicht_in_tags: root = self.mb.props['ORGANON'].xml_tree.getroot() namen = [root.find('.//' + o).attrib['Name'] for o in o_nicht_in_tags ] p = Popup(self.mb, zeit=3) p.text = LANG.LEGE_TAGS_FUER_DATEI_AN + '\n'.join(namen) for o in o_nicht_in_tags: self.erzeuge_tags_ordinal_eintrag(o) # 2) ueberfluessige Tags loeschen if ueberfluessige_tags: pfade = { o : os.path.join(self.mb.pfade['odts'], o + '.odt') for o in ueberfluessige_tags} existierende = { o : pf for o,pf in pfade.items() if os.path.exists(pf)} nicht_existierende = [o for o in ueberfluessige_tags if o not in existierende] # nicht existierende werden ohne Rueckmeldung geloescht for o in nicht_existierende: del self.mb.tags['ordinale'][o] if existierende: text = [] for i,e in enumerate(existierende): if i < 20: text.append(e) else: zaehler = i % 20 text[zaehler] = text[zaehler] + ' ' + e nachricht = LANG.DATEIEN_NICHT_IM_PROJEKT_ABER_AUF_FP.format(self.mb.pfade['odts']) + '\n'.join(text) entscheidung = self.mb.entscheidung(nachricht,"warningbox",16777216) # 3 = Nein oder Cancel, 2 = Ja if entscheidung == 3: return elif entscheidung == 2: for o,pfad in existierende.items(): del self.mb.tags['ordinale'][o] os.remove(pfad) self.mb.class_Bereiche.plain_txt_loeschen(o) except: log(inspect.stack,tb())
def collect(infolder, line = comment_LINE, block = comment_BLOCK, tags = WORDS, marks = MARKS, include=INCLUDE, exclude=EXCLUDE, overwrite=False): # Process block comment marks blocks_open, blocks_close = comment_block_comments(block) # TODO: Make hidden files OS independent, probably using # https://docs.python.org/3.4/library/tempfile.html ? # FIXME: for some reason, if a comment-type ever existed in the TODO # file, but after a while its posts are all gone, the keyword # still remains there, according to the current TODO file, # which still have the "QUESTIONS" keyword, and comment # TODO: Add explicit-remove/browsing capabilities of the .*_cache files # (for example: if git reverted changes --> remove hash from cache file) # The best solution would be a complete CLI tool, to read and manage # and use the cutils command line tools # Compile regular expression patterns pattern1 = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)), blocks_open, r'|'.join(map(comment_escape, tags)), r'|'.join(map(comment_escape, marks)), blocks_close), flags=re_IGNORECASE | re_DOTALL | re_MULTILINE | re_VERBOSE) pattern2 = re_compile(r'\n') # Get previously generated collection of all posts COLLECTED = os_path_join(infolder, '.ccom_todo') try: with open(COLLECTED, 'rb') as file: collected = pickle_load(file) except (FileNotFoundError, EOFError): collected = table_Table(row=OrderedDict) # Clear cache -- remove all non-existing files for filepath in collected.rows(): if not os_path_isfile(filepath): del collected[filepath] # Exception containers except_dirs = [] # relative path to dir from root except_files = [] # relative path to file from root except_names = [] # filename (with extension) anywhere except_exts = [] # extension anywhere # If 'exclude' is dictionary like object try: _empty = () # Exceptions relative to root for key, container in zip(('folders', 'files'), (except_dirs, except_files)): container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty)) # Exceptions anywhere for key, container in zip(('names', 'extensions'), (except_names, except_exts)): container.extend(exclude.get(key, _empty)) # If 'exclude' is an iterable object except AttributeError: except_names = exclude # Include containers permit_names = [] # filename (with extension) anywhere permit_exts = [] # extension anywhere # If 'include' is dictionary like object try: _empty = () # Includes anywhere for key, container in zip(('names', 'extensions'), (permit_names, permit_exts)): container.extend(include.get(key, _empty)) # If 'include' is an iterable object except AttributeError: permit_names = include # Scan through all files and folders with check_Checker(infolder, file='.ccom_cache') as checker: for root, dirs, filenames in os_walk(infolder): # If skip this folder and all subfolders if root in except_dirs: dirs.clear() continue # Check all files in folder for filename in filenames: filepath = os_path_join(root, filename)[2:] # If skip this exact file if filepath in except_files: continue name, extension = os_path_splitext(filename) # If file or extension is not banned and it is on the # white-list and it changed since last time checked and # this is not and overwrite-call if (filename not in except_names and extension not in except_exts and (extension in permit_exts or filename in permit_names) and checker.ischanged(filepath) and not overwrite): with open(filepath, encoding='utf-8') as file: _search(collected, pattern1, pattern2, file.read(), filepath, marks) # Save collection of all posts with open(COLLECTED, 'wb') as file: pickle_dump(collected, file, pickle_HIGHEST_PROTOCOL) # Open the todo file and write out the results with open('TODO', 'w', encoding='utf-8') as todo: # Make it compatible with cver.py todo.write('## INFO ##\n'*2) # Format TODO file as yaml for key in itertools_chain(tags, marks.values()): KEY = key.upper() try: types = collected[KEY].items() len_pos = todo.tell() # Offset for separator comment and # leading and trailing new lines todo.write(' '*82) todo.write('{}:\n'.format(KEY)) index = 1 for filename, posts in types: for i, (linenumber, content) in enumerate(posts, start=index): todo.write(_ITEM.format(msg='\n'.join(content), index=i, short=_SHORT, long=_SHORT*2, sep='- '*38, file=filename, line=linenumber)) index = i + 1 todo.write('\n') # Move back to tag separator comment todo.seek(len_pos) todo.write('\n#{:-^78}#\n'.format( ' {} POSTS IN {} FILES '.format(index - 1, len(types)))) # Move back to the end todo.seek(0, 2) except KeyError: continue print('CCOM: placed {!r}'.format(os_path_join(infolder, 'TODO')))
def an_09984b_anpassen(self): if self.mb.debug: log(inspect.stack) try: # neue Datum Formatierung if self.mb.language == 'de': datum_format = ['dd','mm','yyyy'] else: datum_format = ['mm','dd','yyyy'] self.mb.settings_proj.update({'datum_trenner' : '.', 'datum_format' : datum_format }) self.mb.speicher_settings("project_settings.txt", self.mb.settings_proj) message = u'''This project was created by an older version of Organon. The settings of the tagging category date / time has changed. If your project uses date tags, check if the formatting of dates is in the right order. Standard Organon date formatting is day/month/year for the german version of Organon and month/day/year for any other language version of Organon. The formatting can be set under: Organon menu / File / Settings / Tags / Date Format A backup of your project with the old settings will be created in the backup folder of your project. ''' Popup(self.mb, 'info').text = message self.mb.erzeuge_Backup() # dict sidebar_content in dict tag ueberfuehren pfad = os.path.join(self.mb.pfade['files'],'sidebar_content.pkl') pfad3 = os.path.join(self.mb.pfade['files'],'sidebar_content.pkl.Backup') from pickle import load as pickle_load from pickle import dump as pickle_dump with open(pfad, 'rb') as f: dict_sb_content = pickle_load(f) sb_panels = { 'Synopsis':LANG.SYNOPSIS, 'Notes':LANG.NOTIZEN, 'Images':LANG.BILDER, 'Tags_general':LANG.ALLGEMEIN, 'Tags_characters':LANG.CHARAKTERE, 'Tags_locations':LANG.ORTE, 'Tags_objects':LANG.OBJEKTE, 'Tags_time':LANG.ZEIT, 'Tags_user1':LANG.BENUTZER1, 'Tags_user2':LANG.BENUTZER2, 'Tags_user3':LANG.BENUTZER3 } sb_panels_tup = ( 'Synopsis', 'Notes', 'Images', 'Tags_general', 'Tags_characters', 'Tags_locations', 'Tags_objects', 'Tags_time', 'Tags_user1', 'Tags_user2', 'Tags_user3' ) tags = { 'nr_name' : { 0 : [u'SYNOPSIS','txt'], 1 : [u'NOTIZEN','txt'], 2 : [u'BILDER','img'], 3 : [u'ALLGEMEIN','tag'], 4 : [u'CHARAKTERE','tag'], 5 : [u'ORTE','tag'], 6 : [u'OBJEKTE','tag'], 7 : [u'DATUM','date'], 8 : [u'ZEIT','time'], 9 : [u'BENUTZER1','tag'], 10 : [u'BENUTZER2','tag'], 11 : [u'BENUTZER3','tag'] },} alte_kats = list(dict_sb_content['ordinal'][ list(dict_sb_content['ordinal'])[0] ]) name_index = { k : sb_panels_tup.index(k) for k in alte_kats if k in sb_panels_tup } index_name = { sb_panels_tup.index(k) : k for k in alte_kats if k in sb_panels_tup } tags['ordinale'] = { ordin: {name_index[k] : i2 for k,i2 in i.items()} for ordin,i in dict_sb_content['ordinal'].items() if isinstance(i, dict)} tags['sichtbare'] = [name_index[k] for k in dict_sb_content['sichtbare'] ] tags['sammlung'] = {name_index[k]:i for k,i in dict_sb_content['tags'].items() } tags['nr_name'] = { i : [ getattr(LANG,k[0]) , k[1] ] for i,k in tags['nr_name'].items()} tags['name_nr'] = { k[0] : i for i,k in tags['nr_name'].items()} tags['abfolge'] = list(range(len(tags['nr_name']))) tags['nr_breite'] = {i:2 for i in range(12)} tags['nr_breite'].update({ 0 : 5, 1 : 5, 2 : 3 }) # Tags in Tags_Allgemein loeschen, die in anderen Tags vorhanden sind from itertools import chain alle_tags_in_anderen_panels = list(chain.from_iterable( [v for i,v in tags['sammlung'].items() if i != 3 ] )) for ordi in tags['ordinale']: for t in alle_tags_in_anderen_panels: if t in tags['ordinale'][ordi][3]: tags['ordinale'][ordi][3].remove(t) if t in tags['sammlung'][3]: tags['sammlung'][3].remove(t) # Zeit und Datum trennen for ordi in tags['ordinale']: for i in range(11,8,-1): tags['ordinale'][ordi].update({i:tags['ordinale'][ordi][i-1]}) if 'zeit' in tags['ordinale'][ordi][7]: tags['ordinale'][ordi][8] = tags['ordinale'][ordi][7]['zeit'] else: tags['ordinale'][ordi][8] = None if 'datum' not in tags['ordinale'][ordi][7]: tags['ordinale'][ordi][7] = None elif tags['ordinale'][ordi][7]['datum'] == None: tags['ordinale'][ordi][7] = None else: dat_split = tags['ordinale'][ordi][7]['datum'].split('.') tags['ordinale'][ordi][7] = { datum_format[0] : dat_split[0], datum_format[1] : dat_split[1], datum_format[2] : dat_split[2], } for i in range(11,8,-1): tags['sammlung'].update({i:tags['sammlung'][i-1]}) del tags['sammlung'][8] # neue zeit formatierung panel_nr = [i for i,v in tags['nr_name'].items() if v[1] == 'time'][0] for ordi in tags['ordinale']: zeit = tags['ordinale'][ordi][panel_nr] if zeit == None: continue zeit_str = str(zeit) if len(zeit_str) == 7: zeit_str = '0' + zeit_str std = int(zeit_str[0:2]) minu = int(zeit_str[2:4]) tags['ordinale'][ordi][panel_nr] = '{0}:{1}'.format(std,minu) # tags speichern pfad2 = os.path.join(self.mb.pfade['settings'],'tags.pkl') with open(pfad2, 'wb') as f: pickle_dump(tags, f,2) try: os.remove(pfad) except: pass try: os.remove(pfad3) except: pass except: log(inspect.stack,tb())
def step15( datadir, subsample ): data_ = []; classlabels = []; with open( datadir+'/step07_'+subsample+'.pickle', 'rb' ) as f: data__ = pickle_load( f ); posdims = []; negdims = []; for dimcluster in DIMCLUSTERs: for dim in dimcluster: if dim < 0: negdims.append( abs(dim)-1 ); else: posdims.append( abs(dim)-1 ); rowcount = 0; poscnt = 0; negcnt = 0; for ( y, x ) in data__: rowcount += 1; if rowcount > 2500: break; row = []; for (dim,xval) in enumerate(x): if dim in posdims: row.append( xval ); elif dim in negdims: # row.append( -xval ); row.append( xval ); else: #print( "skipping dim", dim ); row.append( xval ); data_.append( row ); if y == '0': classlabels.append( 0 ); negcnt += 1; elif y == '1': classlabels.append( 1 ); poscnt += 1; data_ = np.array( data_ ); classlabels = np.array( classlabels ); ratio = float( poscnt ) / float( poscnt+negcnt ); data = KernelPCA( n_components=3, kernel='cosine' ).fit_transform( data_ ); print( np.cov(data.T) ); print( data[0] ); plottable_pos = []; plottable_neg = []; for i in range(0,3): plottable_pos.append( [] ); plottable_neg.append( [] ); for (y,x) in zip( classlabels, data ): for (i,xval) in enumerate(x): if y == 0: plottable_neg[ i ].append( xval ); elif y == 1: plottable_pos[ i ].append( xval ); ( fig, ax ) = plt.subplots( nrows = 3, ncols = 3, figsize = ( 3*3, 3*3 ) ); for i in range( 0, 3 ): for j in range( 0, 3 ): if j >= i: continue; ax[i,j].plot( plottable_neg[i], plottable_neg[j], marker='o', color='b', linestyle='', alpha=0.66 ); ax[i,j].plot( plottable_pos[i], plottable_pos[j], marker='o', color='r', linestyle='', alpha=0.66 ); ax[i,j].set_title( str((i,j)) ); fig.savefig( datadir+'/step20_'+subsample+'.png' );
def main(args=None): if args is None: args = sys.argv[1:] np.seterr(all='raise') parser, ns, args = init_args(description='Predict label for unlabeled sequences', args=args) parser = hmmer_args(parser) parser.add_argument('MODEL', type=PathType) parser.add_argument('SEQUENCES', type=PathType) ARGS = parse_args(parser, args, namespace=ns) with gzip_open(ARGS.MODEL, 'rb') as fh: try: model = pickle_load(fh) if model[0] != MODEL_VERSION: raise ImportError('incompatible model version') ARGS.ENCODER, ARGS.LABEL, hmm, extractor, clf = model[1:] except ImportError: msg = 'your model is not of the appropriate version, please re-learn your model' raise RuntimeError(msg) # create a temporary file wherein space characters have been removed with open(ARGS.SEQUENCES) as seq_fh: def seqrecords(): is_dna = ARGS.ENCODER == DNAEncoder seq_fmt = seqfile_format(ARGS.SEQUENCES) source = Verifier(SeqIO.parse(seq_fh, seq_fmt), DNAAlphabet) try: for record in source: yield record if is_dna else translate(record) except VerifyError: if is_dna: msg = ( "your model specifies a DNA encoding " "which is incompatible with protein sequences" ) raise RuntimeError(msg) source.set_alphabet(AminoAlphabet) for record in source: yield record try: fd, tmphmm = mkstemp(); close(fd) with open(tmphmm, 'wb') as hmm_fh: hmm_fh.write(hmm) # explicitly gc hmm hmm = None tmpaln = generate_alignment_(seqrecords(), tmphmm, ARGS) alignment = load_stockholm(tmpaln, trim=True) finally: if exists(tmphmm): remove(tmphmm) if exists(tmpaln): remove(tmpaln) X = extractor.transform(alignment) y = clf.predict(X) feature_names = extractor.get_feature_names() support = clf.named_steps['mrmr'].support_ labels = ['"{0:s}"'.format(feature_names[i]) for i, s in enumerate(support) if s] emptys = [' ' * (len(label) + 2) for label in labels] idlen = max(len(r.id) for r in alignment) + 3 print('{{\n "label": "{0:s}",\n "predictions": ['.format(ARGS.LABEL), file=ARGS.OUTPUT) for i, r in enumerate(alignment): if i > 0: print(',') features = ['[ '] for j, x in enumerate(X[i, support]): if x: features.append(labels[j]) features.append(', ') else: features.append(emptys[j]) features.append(' ]') # replace the last comma with a space idx = None for k, f in enumerate(features): if f == ', ': idx = k if idx is None: features[0] = features[0].rstrip() features[-1] = features[-1].lstrip() else: features[idx] = '' features_ = ''.join(features) print( ' {{{{ "id": {{0:<{0:d}s}} "value": {{1: d}}, "features": {{2:s}} }}}}'.format( idlen).format('"{0:s}",'.format(r.id), y[i], features_), file=ARGS.OUTPUT, end='') print('\n ]\n}', file=ARGS.OUTPUT) finalize_args(ARGS) return 0