def save_diff_in_db(self, files):
"""
Saves Diff, used Unix features: diff.
Args: `files`
Return: None
"""
file1, file2 = files
f_sout = open_file(self.tempfile_patch_u, "w")
if BlackListCompacter.init:
BlackListCompacter.init = 0
subprocess.call("diff -u " + file1 + " " + file2,
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
f_sout.close()
self.save_file_in_db(self.marker_db_init,
open_file(self.tempfile_patch_u, 'r').read())
LOGGER.debug(' marker init in db:%s ', self.marker_db_init)
else:
subprocess.call("diff -u " + file1 + " " + file2,
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
f_sout.close()
self.save_file_in_db(self.marker_db_diff,
open_file(self.tempfile_patch_u, 'r').read())
LOGGER.debug('marker in period in db :%s ', self.marker_db_diff)
def open_csv_file(file, mode=None, **open_kwargs):
"""
Open a file suitable for a CSV reader's source or writer's target.
"""
if sys.version_info[0] < 3: #3--
# Python 2: #3--
if mode is None: #3--
mode = 'rb' #3--
return open_file(file, mode, **open_kwargs) #3--
# Python 3:
if mode is None:
mode = 'r'
elif 'b' in mode:
raise ValueError("the given mode ({0!a}) is binary (which is "
"not suitable for a CSV reader's source or "
"writer's target in Python 3)".format(mode))
# Important: CSV source/target file should be opened with newline=''
# (see: https://docs.python.org/3/library/csv.html#module-contents).
return open_file(file, mode, newline='', **open_kwargs)
def backup_msg(fname, collection, msg, header):
with open_file(fname, 'wb') as f:
if isinstance(msg, (bytes, str)):
payload = (msg.encode('utf-8') if isinstance(msg, str)
else msg)
else:
payload = (ascii(msg).encode('utf-8') if isinstance(ascii(msg), str)
else ascii(msg))
hdr = (ascii(header).encode('utf-8') if isinstance(ascii(header), str)
else ascii(header))
f.write('\n'.join(( collection, hdr, payload )))
def init_files(self):
"""Init all tmp files"""
self.tempfilefd_file_init, self.tempfile_file_init = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_file, self.tempfile_file = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_patch_all, self.tempfile_patch_all = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_patch, self.tempfile_patch = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_patch_tmp, self.tempfile_patch_tmp = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_patch_u, self.tempfile_patch_u = tempfile.mkstemp(
self.prefix, self.suffix)
(self.tempfilefd_file_recovery_0,
self.tempfile_file_recovery_0) = tempfile.mkstemp(
self.prefix, self.suffix)
self.tempfilefd_ver, self.tempfile_ver = tempfile.mkstemp(
self.prefix, self.suffix)
self.list_tmp_files.append(
(self.tempfilefd_file_init, self.tempfile_file_init))
self.list_tmp_files.append((self.tempfilefd_file, self.tempfile_file))
self.list_tmp_files.append(
(self.tempfilefd_patch_all, self.tempfile_patch_all))
self.list_tmp_files.append(
(self.tempfilefd_patch_tmp, self.tempfile_patch_tmp))
self.list_tmp_files.append(
(self.tempfilefd_patch_u, self.tempfile_patch_u))
self.list_tmp_files.append(
(self.tempfilefd_file_recovery_0, self.tempfile_file_recovery_0))
self.list_tmp_files.append(
(self.tempfilefd_patch, self.tempfile_patch))
self.list_tmp_files.append((self.tempfilefd_ver, self.tempfile_ver))
# save orig init file
with open_file(self.tempfile_file_init, 'w') as fid:
LOGGER.debug('WTF: %r', type(self.payload))
fid.write(self.payload)
self.file_init = self.tempfile_file_init
for fd, fn in self.list_tmp_files:
os.close(fd)
os.chmod(fn, 0o644)
LOGGER.debug('run blacklist init tmp files')
def backup_msg(fname, collection, msg, header):
with open_file(fname, 'w') as f: #3: add 'b'-mode flag
if isinstance(msg, (bytes, unicode)): #3: unicode->str
payload = (
msg.encode('utf-8')
if isinstance(msg, unicode) #3: unicode->str
else msg)
else:
payload = (
repr(msg).encode('utf-8')
if isinstance(repr(msg), unicode) #3: unicode->str
else repr(msg))
hdr = (
repr(header).encode('utf-8')
if isinstance(repr(header), unicode) #3: unicode->str
else repr(header))
f.write('\n'.join((collection, hdr, payload)))
def open_csv_file(file, mode='r', **open_kwargs):
"""
Open a file suitable for a CSV reader's source or writer's target.
Args/kwargs:
`file`:
Typically it is a string specifying the name (path) of the
file to be opened. For more information, see the docs of the
built-in function `open()`.
`mode` (default: `'r'`):
An optional string that specifies the mode in which the file
is opened. It should *not* contain `'b'` as then it would
specify a binary mode which is inappropriate for CSV readers
and writers. See also: the docs of the built-in function
`open()`.
Other optional arguments, only as *keyword* (named) ones:
See the docs of `n6lib.common_helpers.open_file()`, except
that the `newline` argument should *not* be given (because
it will be automatically set to `''`, as that is the
appropriate setting for CSV readers and writers; see also:
the docs of the Python's standard library module `csv`.
Returns:
A file object (for details, see the docs of the built-in
function `open()`).
Raises:
`ValueError`:
if the `mode` argument specifies a binary mode (i.e.,
contains `'b'`).
`TypeError`:
if the `newline` argument is given.
See also: the docs of the built-in function `open()`.
"""
if 'b' in mode:
raise ValueError(f"the given mode ({mode!a}) is binary (which "
f"is not suitable for a CSV reader's source "
f"or writer's target)")
# Important: CSV source/target file should be opened with newline=''
# (see: https://docs.python.org/3/library/csv.html#module-contents).
return open_file(file, mode, newline='', **open_kwargs)
def restore_state(self):
with open_file(self.dbpath, "rb") as f:
self.comp_data = pickle.load(f)
def store_state(self):
try:
with open_file(self.dbpath, "wb") as f:
pickle.dump(self.comp_data, f, 2)
except IOError:
LOGGER.error("Error saving state to: %r", self.dbpath)
def load_template():
global CONF_TEMPLATE
with open_file('program_template.tmpl', 'r') as f:
CONF_TEMPLATE = f.read()
def create_supervisord_conf(program_name):
with open_file('programs/%s.conf' % program_name, "w") as f:
f.write(CONF_TEMPLATE.format(program_name=program_name, program_command=program_name))
def restore_state(self):
with open_file(self.dbpath, 'rb') as f:
self.aggr_data = pickle.load(f)
def store_state(self):
try:
with open_file(self.dbpath, 'wb') as f:
pickle.dump(self.aggr_data, f, protocol=2)
except IOError:
LOGGER.error('Error saving state to: %r', self.dbpath)
def generate_orig_file(self, cursor, file_id):
"""
Generates one or more files, patching one file to another.
Used Unix features: patch.
Args: `cursor`: (with all the patch from one period)
`file_id`: first init file id to generate first patch
Return: None
"""
LOGGER.debug('BlackListCompacter.GENERATE_ALL_FILE: %r',
BlackListCompacter.generate_all_file)
# generate first file
files_count = 1
# stdout in file
f_sout = open_file(self.tempfile_patch_u, "w")
# first diff file post init in GENERATE_ALL_FILE mode
if cursor.count() > 0 and BlackListCompacter.generate_all_file:
out = subprocess.call("patch " + self.tempfile_file + " -i " +
self.tempfile_patch_tmp + " -o " +
self.tempfile_ver + str(files_count - 1),
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
LOGGER.debug('patch_next_file(return code): %r', out)
self.list_tmp_files.append(
(self.tempfile_ver, self.tempfile_ver + str(files_count - 1)))
os.chmod(self.tempfile_ver + str(files_count - 1), 0o644)
# # first diff file post init in GENERATE_ONE_FILE mode
elif cursor.count() > 0 and (not BlackListCompacter.generate_all_file):
out = subprocess.call("patch " + self.tempfile_file + " -i " +
self.tempfile_patch_tmp + " -o " +
self.tempfile_file_recovery_0,
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
LOGGER.debug('patch_next_file(return code): %r', out)
else:
file_db = self.dbm.get_file_from_db_raw(file_id)
patch_file = open_file(self.tempfile_patch_tmp, 'w')
patch_file.write(file_db)
patch_file.close()
out = subprocess.call("patch " + self.tempfile_file_recovery_0 +
" -i " + self.tempfile_patch_tmp,
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
LOGGER.debug('patch_first_file(return code): %r', out)
for i in cursor:
id_dba = i["_id"]
# set prev id in current doc.
self.prev_id = id_dba
file_db = self.dbm.get_file_from_db_raw(id_dba)
patch_file = open_file(self.tempfile_patch_tmp, 'w')
patch_file.write(file_db)
patch_file.close()
# # gen. tmp all version files
self.list_tmp_files.append(
(self.tempfilefd_ver, self.tempfile_ver + str(files_count)))
if BlackListCompacter.generate_all_file:
# # generate all partial files
out = subprocess.call("patch " + self.tempfile_ver +
str(files_count - 1) + " -i " +
self.tempfile_patch_tmp + " -o " +
self.tempfile_ver + str(files_count),
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
os.chmod(self.tempfile_ver + str(files_count), 0o644)
LOGGER.debug('patch_all_files(return code): %r', out)
else:
out = subprocess.call("patch " +
self.tempfile_file_recovery_0 + " -i " +
self.tempfile_patch_tmp,
stdout=f_sout,
stderr=subprocess.STDOUT,
shell=True)
LOGGER.debug('patch(return code): %r', out)
files_count += 1
f_sout.close()
return self.tempfile_file_recovery_0
