def save_diff_in_db(self, files): """ Saves Diff, used Unix features: diff. Args: `files` Return: None """ file1, file2 = files f_sout = open_file(self.tempfile_patch_u, "w") if BlackListCompacter.init: BlackListCompacter.init = 0 subprocess.call("diff -u " + file1 + " " + file2, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) f_sout.close() self.save_file_in_db(self.marker_db_init, open_file(self.tempfile_patch_u, 'r').read()) LOGGER.debug(' marker init in db:%s ', self.marker_db_init) else: subprocess.call("diff -u " + file1 + " " + file2, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) f_sout.close() self.save_file_in_db(self.marker_db_diff, open_file(self.tempfile_patch_u, 'r').read()) LOGGER.debug('marker in period in db :%s ', self.marker_db_diff)
def open_csv_file(file, mode=None, **open_kwargs): """ Open a file suitable for a CSV reader's source or writer's target. """ if sys.version_info[0] < 3: #3-- # Python 2: #3-- if mode is None: #3-- mode = 'rb' #3-- return open_file(file, mode, **open_kwargs) #3-- # Python 3: if mode is None: mode = 'r' elif 'b' in mode: raise ValueError("the given mode ({0!a}) is binary (which is " "not suitable for a CSV reader's source or " "writer's target in Python 3)".format(mode)) # Important: CSV source/target file should be opened with newline='' # (see: https://docs.python.org/3/library/csv.html#module-contents). return open_file(file, mode, newline='', **open_kwargs)
def backup_msg(fname, collection, msg, header): with open_file(fname, 'wb') as f: if isinstance(msg, (bytes, str)): payload = (msg.encode('utf-8') if isinstance(msg, str) else msg) else: payload = (ascii(msg).encode('utf-8') if isinstance(ascii(msg), str) else ascii(msg)) hdr = (ascii(header).encode('utf-8') if isinstance(ascii(header), str) else ascii(header)) f.write('\n'.join(( collection, hdr, payload )))
def init_files(self): """Init all tmp files""" self.tempfilefd_file_init, self.tempfile_file_init = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_file, self.tempfile_file = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_patch_all, self.tempfile_patch_all = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_patch, self.tempfile_patch = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_patch_tmp, self.tempfile_patch_tmp = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_patch_u, self.tempfile_patch_u = tempfile.mkstemp( self.prefix, self.suffix) (self.tempfilefd_file_recovery_0, self.tempfile_file_recovery_0) = tempfile.mkstemp( self.prefix, self.suffix) self.tempfilefd_ver, self.tempfile_ver = tempfile.mkstemp( self.prefix, self.suffix) self.list_tmp_files.append( (self.tempfilefd_file_init, self.tempfile_file_init)) self.list_tmp_files.append((self.tempfilefd_file, self.tempfile_file)) self.list_tmp_files.append( (self.tempfilefd_patch_all, self.tempfile_patch_all)) self.list_tmp_files.append( (self.tempfilefd_patch_tmp, self.tempfile_patch_tmp)) self.list_tmp_files.append( (self.tempfilefd_patch_u, self.tempfile_patch_u)) self.list_tmp_files.append( (self.tempfilefd_file_recovery_0, self.tempfile_file_recovery_0)) self.list_tmp_files.append( (self.tempfilefd_patch, self.tempfile_patch)) self.list_tmp_files.append((self.tempfilefd_ver, self.tempfile_ver)) # save orig init file with open_file(self.tempfile_file_init, 'w') as fid: LOGGER.debug('WTF: %r', type(self.payload)) fid.write(self.payload) self.file_init = self.tempfile_file_init for fd, fn in self.list_tmp_files: os.close(fd) os.chmod(fn, 0o644) LOGGER.debug('run blacklist init tmp files')
def backup_msg(fname, collection, msg, header): with open_file(fname, 'w') as f: #3: add 'b'-mode flag if isinstance(msg, (bytes, unicode)): #3: unicode->str payload = ( msg.encode('utf-8') if isinstance(msg, unicode) #3: unicode->str else msg) else: payload = ( repr(msg).encode('utf-8') if isinstance(repr(msg), unicode) #3: unicode->str else repr(msg)) hdr = ( repr(header).encode('utf-8') if isinstance(repr(header), unicode) #3: unicode->str else repr(header)) f.write('\n'.join((collection, hdr, payload)))
def open_csv_file(file, mode='r', **open_kwargs): """ Open a file suitable for a CSV reader's source or writer's target. Args/kwargs: `file`: Typically it is a string specifying the name (path) of the file to be opened. For more information, see the docs of the built-in function `open()`. `mode` (default: `'r'`): An optional string that specifies the mode in which the file is opened. It should *not* contain `'b'` as then it would specify a binary mode which is inappropriate for CSV readers and writers. See also: the docs of the built-in function `open()`. Other optional arguments, only as *keyword* (named) ones: See the docs of `n6lib.common_helpers.open_file()`, except that the `newline` argument should *not* be given (because it will be automatically set to `''`, as that is the appropriate setting for CSV readers and writers; see also: the docs of the Python's standard library module `csv`. Returns: A file object (for details, see the docs of the built-in function `open()`). Raises: `ValueError`: if the `mode` argument specifies a binary mode (i.e., contains `'b'`). `TypeError`: if the `newline` argument is given. See also: the docs of the built-in function `open()`. """ if 'b' in mode: raise ValueError(f"the given mode ({mode!a}) is binary (which " f"is not suitable for a CSV reader's source " f"or writer's target)") # Important: CSV source/target file should be opened with newline='' # (see: https://docs.python.org/3/library/csv.html#module-contents). return open_file(file, mode, newline='', **open_kwargs)
def restore_state(self): with open_file(self.dbpath, "rb") as f: self.comp_data = pickle.load(f)
def store_state(self): try: with open_file(self.dbpath, "wb") as f: pickle.dump(self.comp_data, f, 2) except IOError: LOGGER.error("Error saving state to: %r", self.dbpath)
def load_template(): global CONF_TEMPLATE with open_file('program_template.tmpl', 'r') as f: CONF_TEMPLATE = f.read()
def create_supervisord_conf(program_name): with open_file('programs/%s.conf' % program_name, "w") as f: f.write(CONF_TEMPLATE.format(program_name=program_name, program_command=program_name))
def restore_state(self): with open_file(self.dbpath, 'rb') as f: self.aggr_data = pickle.load(f)
def store_state(self): try: with open_file(self.dbpath, 'wb') as f: pickle.dump(self.aggr_data, f, protocol=2) except IOError: LOGGER.error('Error saving state to: %r', self.dbpath)
def generate_orig_file(self, cursor, file_id): """ Generates one or more files, patching one file to another. Used Unix features: patch. Args: `cursor`: (with all the patch from one period) `file_id`: first init file id to generate first patch Return: None """ LOGGER.debug('BlackListCompacter.GENERATE_ALL_FILE: %r', BlackListCompacter.generate_all_file) # generate first file files_count = 1 # stdout in file f_sout = open_file(self.tempfile_patch_u, "w") # first diff file post init in GENERATE_ALL_FILE mode if cursor.count() > 0 and BlackListCompacter.generate_all_file: out = subprocess.call("patch " + self.tempfile_file + " -i " + self.tempfile_patch_tmp + " -o " + self.tempfile_ver + str(files_count - 1), stdout=f_sout, stderr=subprocess.STDOUT, shell=True) LOGGER.debug('patch_next_file(return code): %r', out) self.list_tmp_files.append( (self.tempfile_ver, self.tempfile_ver + str(files_count - 1))) os.chmod(self.tempfile_ver + str(files_count - 1), 0o644) # # first diff file post init in GENERATE_ONE_FILE mode elif cursor.count() > 0 and (not BlackListCompacter.generate_all_file): out = subprocess.call("patch " + self.tempfile_file + " -i " + self.tempfile_patch_tmp + " -o " + self.tempfile_file_recovery_0, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) LOGGER.debug('patch_next_file(return code): %r', out) else: file_db = self.dbm.get_file_from_db_raw(file_id) patch_file = open_file(self.tempfile_patch_tmp, 'w') patch_file.write(file_db) patch_file.close() out = subprocess.call("patch " + self.tempfile_file_recovery_0 + " -i " + self.tempfile_patch_tmp, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) LOGGER.debug('patch_first_file(return code): %r', out) for i in cursor: id_dba = i["_id"] # set prev id in current doc. self.prev_id = id_dba file_db = self.dbm.get_file_from_db_raw(id_dba) patch_file = open_file(self.tempfile_patch_tmp, 'w') patch_file.write(file_db) patch_file.close() # # gen. tmp all version files self.list_tmp_files.append( (self.tempfilefd_ver, self.tempfile_ver + str(files_count))) if BlackListCompacter.generate_all_file: # # generate all partial files out = subprocess.call("patch " + self.tempfile_ver + str(files_count - 1) + " -i " + self.tempfile_patch_tmp + " -o " + self.tempfile_ver + str(files_count), stdout=f_sout, stderr=subprocess.STDOUT, shell=True) os.chmod(self.tempfile_ver + str(files_count), 0o644) LOGGER.debug('patch_all_files(return code): %r', out) else: out = subprocess.call("patch " + self.tempfile_file_recovery_0 + " -i " + self.tempfile_patch_tmp, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) LOGGER.debug('patch(return code): %r', out) files_count += 1 f_sout.close() return self.tempfile_file_recovery_0