def _tar_config_files(self, tarfile):
import glob
import os
for config_file in glob.glob('*.config.yaml'):
tarfile.add(config_file,
arcname=os.path.join(self._job_name, config_file))
def createArchive(self, cfgOutName, name='', mode='w:gz'):
"""
create the archive to upload
"""
if not name:
import uuid
name = os.path.join(os.getcwd(), str(uuid.uuid4()) + 'default.tgz')
import tarfile
print 'opening tar file'
tarfile = tarfile.open(name=name, mode=mode, dereference=True)
print 'adding %s to the tarball' % cfgOutName
tarfile.add(cfgOutName, arcname='PSet.py')
#checkSum
print 'calculating the checksum'
lsl = [(x.name, int(x.size), int(x.mtime), x.uname)
for x in tarfile.getmembers()]
# hasher = hashlib.md5(str(lsl))
hasher = hashlib.sha256(str(lsl))
checksum = hasher.hexdigest()
#end
tarfile.close()
return name, checksum
def _tar_resources(self, tarfile):
import os
module_directory = os.path.dirname(os.path.abspath(__file__))
resource_directory = os.path.join(module_directory, "resources")
os.chdir(resource_directory)
tarfile.add(".", arcname=self._job_name)
def createArchive(self, cfgOutName , name = '', mode = 'w:gz'):
"""
create the archive to upload
"""
if not name:
import uuid
name = os.path.join(os.getcwd(), str(uuid.uuid4()) +'default.tgz')
import tarfile
print 'opening tar file'
tarfile = tarfile.open(name=name , mode=mode, dereference=True)
print 'adding %s to the tarball' % cfgOutName
tarfile.add(cfgOutName, arcname='PSet.py')
#checkSum
print 'calculating the checksum'
lsl = [(x.name, int(x.size), int(x.mtime), x.uname) for x in tarfile.getmembers()]
# hasher = hashlib.md5(str(lsl))
hasher = hashlib.sha256(str(lsl))
checksum = hasher.hexdigest()
#end
tarfile.close()
return name, checksum
def pack_into(self, tarfile):
for (dirpath, _, filenames) in os.walk(self.__source_dir):
rel_dir = os.path.normpath(os.path.relpath(dirpath, self.__source_dir))
for filename in filenames:
if not re.match('.*\.h$', filename) is None:
full_src = os.path.join(self.__source_dir, rel_dir, filename)
full_tgt = os.path.join(self.__target_dir, rel_dir, filename)
tarfile.add(name = full_src, arcname = full_tgt)
def recursively_add_directory(tarfile, directory_path, path_in_archive, exclude=None):
for sub_path, dirs, files in os.walk(directory_path):
assert sub_path[:len(directory_path)]==directory_path
relative_dir = sub_path[len(directory_path):]
if len(relative_dir)>0 and relative_dir[0] in [os.path.sep, os.path.altsep]:
relative_dir = relative_dir[1:]
for fname in files:
if exclude is None or not exclude(fname):
tarfile.add(path.join(sub_path, fname), path.join(path_in_archive, relative_dir, fname))
def recursively_add_directory(tarfile, directory_path, path_in_archive, exclude=None):
for sub_path, dirs, files in os.walk(directory_path):
assert sub_path[: len(directory_path)] == directory_path
relative_dir = sub_path[len(directory_path) :]
if len(relative_dir) > 0 and relative_dir[0] in [os.path.sep, os.path.altsep]:
relative_dir = relative_dir[1:]
for fname in files:
if exclude is None or not exclude(fname):
tarfile.add(path.join(sub_path, fname), path.join(path_in_archive, relative_dir, fname))
def add_meta_files(tarfile, meta_files_dir):
"""Adds the meta files to the specified tarfile.
Args:
tarfile: The tarfile object where the files needs to be added.
meta_files_dir: The directory containing the meta files.
"""
tarfile.add(os.path.join(meta_files_dir, 'TIMESTAMP'), arcname='TIMESTAMP')
tarfile.add(os.path.join(meta_files_dir, 'SCHEMA_SEQUENCE'), arcname='SCHEMA_SEQUENCE')
def _tar_env(self, tarfile):
import os
from foundations_contrib.simple_tempfile import SimpleTempfile
from foundations_contrib.job_bundling.script_environment import ScriptEnvironment
with SimpleTempfile('w+') as temp_file:
ScriptEnvironment(self._config).write_environment(temp_file)
tarfile.add(temp_file.name,
arcname=os.path.join(self._job_name, 'run.env'))
def _tar_foundations_modules(self, tarfile):
from foundations_internal.global_state import module_manager
import os
for module_name, module_directory in module_manager.module_directories_and_names(
):
self._log().debug('Adding module {} at {}'.format(
module_name, module_directory))
tarfile.add(module_directory,
arcname=self._job_name + os.path.sep + module_name)
def bk_now(self, tables=None):
'''
使用逻辑备份导出指定用户的数据表，参数tables为表名元组，默认为空时导出整个用户空间。
'''
timestamp = str(
time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
bak_dir = self.bkdir + '\\' + timestamp
dmp_file = bak_dir + '\\' + timestamp + '_exp.dmp'
log_file = bak_dir + '\\' + timestamp + '_exp.log'
options = self.conf['seruser'] + '/' + self.conf['serpass'] + ' buffer=64000 file=' \
+ dmp_file + ' log=' + log_file
if tables is not None:
options += ' tables=' + tables
else:
options += ' owner=' + self.conf['seruser']
os.mkdir(bak_dir)
if os.system('exp ' + options) == 0:
zip_file = bak_dir + '\\' + timestamp + '_exp.zip'
myzipfile = zipfile.ZipFile(zip_file, 'w')
myzipfile.write(dmp_file)
myzipfile.close()
return True, zip_file
else:
print 'backup oprations failed!'
return False, None
elif utils.is_linux():
bak_dir = self.bkdir + '/' + timestamp
dmp_file = bak_dir + '/' + timestamp + '_exp.dmp'
log_file = bak_dir + '/' + timestamp + '_exp.log'
options = self.conf['seruser'] + '/' + self.conf['serpass'] + ' buffer=64000 file=' \
+ dmp_file + ' log=' + log_file
if tables is not None:
options += ' tables=' + tables
else:
options += ' owner=' + self.conf['seruser']
os.mkdir(bak_dir)
if os.system('exp ' + options) == 0:
tar_file = bak_dir + '/' + timestamp + '_exp.tar.gz'
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(dmp_file)
tarfile.close()
return True, tar_file
else:
print 'Unkown System Type!'
sys.exit(1)
def bk_now(self, tables=None):
'''
使用逻辑备份导出指定用户的数据表，参数tables为表名元组，默认为空时导出整个用户空间。
'''
timestamp = str(time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
bak_dir = self.bkdir + '\\' + timestamp
dmp_file = bak_dir + '\\' + timestamp + '_exp.dmp'
log_file = bak_dir + '\\' + timestamp + '_exp.log'
options = self.conf['seruser'] + '/' + self.conf['serpass'] + ' buffer=64000 file=' \
+ dmp_file + ' log=' + log_file
if tables is not None:
options += ' tables=' + tables
else:
options += ' owner=' + self.conf['seruser']
os.mkdir(bak_dir)
if os.system('exp ' + options) == 0:
zip_file = bak_dir + '\\' + timestamp + '_exp.zip'
myzipfile = zipfile.ZipFile(zip_file, 'w')
myzipfile.write(dmp_file)
myzipfile.close()
return True, zip_file
else:
print 'backup oprations failed!'
return False, None
elif utils.is_linux():
bak_dir = self.bkdir + '/' + timestamp
dmp_file = bak_dir + '/' + timestamp + '_exp.dmp'
log_file = bak_dir + '/' + timestamp + '_exp.log'
options = self.conf['seruser'] + '/' + self.conf['serpass'] + ' buffer=64000 file=' \
+ dmp_file + ' log=' + log_file
if tables is not None:
options += ' tables=' + tables
else:
options += ' owner=' + self.conf['seruser']
os.mkdir(bak_dir)
if os.system('exp ' + options) == 0:
tar_file = bak_dir + '/' + timestamp + '_exp.tar.gz'
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(dmp_file)
tarfile.close()
return True, tar_file
else:
print 'Unkown System Type!'
sys.exit(1)
def glob_bak(self):
timestamp = str(
time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
dump_dir = self.bkdir + '\\' + timestamp
dump_file = dump_dir + '\\' + timestamp + '_glob.sql'
zip_file_path = dump_dir + '\\' + timestamp + '_glob.zip'
log_file = dump_dir + '\\' + timestamp + '_glob.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' --all-databases --flush-logs --master-data=2'
os.mkdir(dump_dir)
os.system('purge master logs')
if os.system('mysqldump ' + options + ' > ' + dump_file) == 0:
myzipfile = zipfile.ZipFile(zip_file_path, 'w')
myzipfile.write(dump_file)
myzipfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, zip_file_path
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
elif utils.is_linux():
dump_dir = self.bkdir + '/' + timestamp
dump_file = dump_dir + '/' + timestamp + '_glob.sql'
tar_file = dump_dir + '/' + timestamp + '_glob.tar.gz'
log_file = dump_dir + '/' + timestamp + '_glob.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' --all-databases --flush-logs --master-data=2'
os.mkdir(dump_dir)
os.system('purge master logs')
if os.system('mysqldump ' + options + ' > ' + dump_file) == 0:
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(dump_file)
tarfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, tar_file
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
else:
print 'Unkown System Type!'
sys.exit(1)
def incr_bak(self):
timestamp = str(
time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
dump_dir = self.bkdir + '\\' + timestamp
zip_file_path = dump_dir + '\\' + timestamp + '_incr.zip'
log_file = dump_dir + '\\' + timestamp + '_incr.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' flush-logs'
os.mkdir(dump_dir)
if os.system('mysqladmin ' + options) == 0:
logbin_file = self._get_logbin_last()
print logbin_file
myzipfile = zipfile.ZipFile(zip_file_path, 'w')
myzipfile.write(logbin_file)
myzipfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, zip_file_path
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
elif utils.is_linux():
dump_dir = self.bkdir + '/' + timestamp
tar_file = dump_dir + '/' + timestamp + '_incr.tar.gz'
log_file = dump_dir + '/' + timestamp + '_incr.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' flush-logs'
os.mkdir(dump_dir)
if os.system('mysqladmin ' + options) == 0:
logbin_file = self._get_logbin_last()
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(logbin_file)
tarfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, tar_file
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
else:
print 'Unkown System Type!'
sys.exit(1)
pass
def incr_bak(self):
timestamp = str(time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
dump_dir = self.bkdir + '\\' + timestamp
zip_file_path = dump_dir + '\\' + timestamp + '_incr.zip'
log_file = dump_dir + '\\' + timestamp + '_incr.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' flush-logs'
os.mkdir(dump_dir)
if os.system('mysqladmin ' + options) == 0:
logbin_file = self._get_logbin_last()
print logbin_file
myzipfile = zipfile.ZipFile(zip_file_path, 'w')
myzipfile.write(logbin_file)
myzipfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, zip_file_path
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
elif utils.is_linux():
dump_dir = self.bkdir + '/' + timestamp
tar_file = dump_dir + '/' + timestamp + '_incr.tar.gz'
log_file = dump_dir + '/' + timestamp + '_incr.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' flush-logs'
os.mkdir(dump_dir)
if os.system('mysqladmin ' + options) == 0:
logbin_file = self._get_logbin_last()
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(logbin_file)
tarfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, tar_file
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
else:
print 'Unkown System Type!'
sys.exit(1)
pass
def glob_bak(self):
timestamp = str(time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(time.time())))
if utils.is_windows():
dump_dir = self.bkdir + '\\' + timestamp
dump_file = dump_dir + '\\' + timestamp + '_glob.sql'
zip_file_path = dump_dir + '\\' + timestamp + '_glob.zip'
log_file = dump_dir + '\\' + timestamp + '_glob.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' --all-databases --flush-logs --master-data=2'
os.mkdir(dump_dir)
os.system('purge master logs')
if os.system('mysqldump ' + options + ' > ' + dump_file) == 0:
myzipfile = zipfile.ZipFile(zip_file_path, 'w')
myzipfile.write(dump_file)
myzipfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, zip_file_path
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
elif utils.is_linux():
dump_dir = self.bkdir + '/' + timestamp
dump_file = dump_dir + '/' + timestamp + '_glob.sql'
tar_file = dump_dir + '/' + timestamp + '_glob.tar.gz'
log_file = dump_dir + '/' + timestamp + '_glob.log'
options = '-h' + self.conf['serip'] + ' -u' + self.conf['seruser'] \
+ ' -p' + self.conf['serpass'] + ' --all-databases --flush-logs --master-data=2'
os.mkdir(dump_dir)
os.system('purge master logs')
if os.system('mysqldump ' + options + ' > ' + dump_file) == 0:
tarfile = tarfile.open(tar_file, 'w:gz')
tarfile.add(dump_file)
tarfile.close()
os.system('echo "DataBase Backup Success!" >> ' + log_file)
return True, tar_file
else:
os.system('echo "DataBase Backup Failed! >> ' + log_file)
return False, None
else:
print 'Unkown System Type!'
sys.exit(1)
def save(self, tarfile):
print(self.camera_files)
# Save the camera files
for name, file in self.camera_files.items():
f = NamedTemporaryFile(delete=False)
try:
# Convert to jpg 70% quality
w, h, data = convert_image(file, 70)
f.write(data)
f.close()
arcname = "{}/{}.jpg".format(self.index, name)
tarfile.add(f.name, arcname=arcname)
finally:
os.unlink(f.name)
meta = lisa_pb2.Frame()
self.egopose.apply(meta)
print(meta)
print(meta.SerializeToString())
f = NamedTemporaryFile(delete=False)
try:
f.write(meta.SerializeToString())
f.close()
tarfile.add(f.name, arcname="%d/metadata.proto" % (self.index))
tarfile.add(self.pcd_file, arcname="%d/points.pcd" % (self.index))
finally:
os.unlink(f.name)
# Remove duplicates.
files_for_tarball = set(files_for_tarball)
with tarfile.open(os.path.join(html_output_dir, example_name + '.tar.gz'), \
'w:gz', dereference=True) as tarfile:
example_dir_idx = example_dir.index(example_name)
def strip_path(tarinfo):
tarinfo.name = tarinfo.name[example_dir_idx-1:]
# Put the inputs and outputs into the build directory because the test
# will not be able to find them otherwise.
basename = os.path.basename(tarinfo.name)
if basename == 'CMakeLists.txt.tarball':
head, tail = os.path.split(tarinfo.name)
tarinfo.name = os.path.join(head, 'CMakeLists.txt')
return tarinfo
for path in files_for_tarball:
tarfile.add(path, filter=strip_path)
with zipfile.ZipFile(os.path.join(html_output_dir, example_name + '.zip'), \
'w') as zipfile:
example_dir_idx = example_dir.index(example_name)
for path in files_for_tarball:
arcname = path[example_dir_idx-1:]
# Put the inputs and outputs into the build directory because the test
# will not be able to find them otherwise.
basename = os.path.basename(arcname)
if basename == 'CMakeLists.txt.tarball':
head, tail = os.path.split(arcname)
arcname = os.path.join(head, 'CMakeLists.txt')
zipfile.write(path, arcname)
import tarfile as tar
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
content = MIMEMultipart() #建立MIMEMultipart物件
content["subject"] = "Mail title" #郵件標題
content["from"] = "*****@*****.**" #寄件者
content["to"] = "*****@*****.**" #收件者
# 壓縮log檔案
tar = tar.open("file name", "w:gz")
for root, dir, files in os.walk("/var/log"):
for file in files:
if "secure" in file:
fullpath = os.path.join(root, file)
tar.add(fullpath)
tar.close()
# 文字內容
content.attach(MIMEText("hdfs-overcomer.ddns.net secure log files.")) #郵件內容
# 附加檔案
att = MIMEText(open('compress file name', 'rb').read(), 'base64', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
att["Content-Disposition"] = 'attachment; filename="compress file name"'
content.attach(att) #郵件內容
import smtplib
with smtplib.SMTP(host="smtp.gmail.com", port="587") as smtp: # 設定SMTP伺服器
def pack_into(self, tarfile):
tarfile.add(name = self.__source, arcname = self.__target)
def _tar_job_source_bundle_archive(self, tarfile):
import os
tarfile.add(self._job_source_bundle.job_archive(),
arcname=os.path.join(self._job_name,
self._job_source_name))
songname = basename
if trackno.match(songname):
songname = trackno.sub('', songname)
albumdir = os.path.basename(dirpath)
artistdir = os.path.basename(os.path.dirname(dirpath))
key = (artistdir, albumdir, songname, dirpath)
value = os.path.join(dirpath, filename)
found[key].append((index, value))
count += 1
artists = sorted(set([x[0] for x in found]))
albums = sorted(set([x[:2] for x in found]))
# re-do everything by artist/album
by_artist_album = collections.defaultdict(dict)
for artist, album in albums:
for key in found:
if key[0] == artist and key[1] == album:
by_artist_album[(artist, album)][key[2]] = found[key]
for (artist, album), songs in by_artist_album.items():
# figure the albumdir
albumdir = os.path.dirname(songs[songs.keys()[0]][0][1])
archive_path = os.path.join(albumdir, '@'.join([re.sub('[^-A-Za-z0-9_]', '_', artist), re.sub('[^-A-Za-z0-9_]', '_', album)]) + ".tar.bz2")
tarfile = tarfile.open(archive_path, mode='w:bz2')
for songname, locations in songs.items():
for idx, location in locations:
tarfile.add(location, arcname=songname+".mp3")
rmfile.write("%s\n" % (location,))
tarfile.close()
print archive_path
def _tar_job_binary(self, tarfile):
import os
tarfile.add(self._job_binary(),
arcname=os.path.join(self._job_name, self._job_binary()))
inputs = glob.glob(os.path.join(example_dir, '*.md5'))
for path in inputs:
files_for_tarball.append(path[:-4])
with tarfile.open(os.path.join(html_output_dir, example_name + '.tar.gz'), \
'w:gz', dereference=True) as tarfile:
example_dir_idx = example_dir.index(example_name)
def strip_path(tarinfo):
tarinfo.name = tarinfo.name[example_dir_idx-1:]
# Put the inputs and outputs into the build directory because the test
# will not be able to find them otherwise.
basename = os.path.basename(tarinfo.name)
if basename == 'CMakeLists.txt.tarball':
head, tail = os.path.split(tarinfo.name)
tarinfo.name = os.path.join(head, 'CMakeLists.txt')
return tarinfo
for path in files_for_tarball:
tarfile.add(path, filter=strip_path)
with zipfile.ZipFile(os.path.join(html_output_dir, example_name + '.zip'), \
'w') as zipfile:
example_dir_idx = example_dir.index(example_name)
for path in files_for_tarball:
arcname = path[example_dir_idx-1:]
# Put the inputs and outputs into the build directory because the test
# will not be able to find them otherwise.
basename = os.path.basename(arcname)
if basename == 'CMakeLists.txt.tarball':
head, tail = os.path.split(arcname)
arcname = os.path.join(head, 'CMakeLists.txt')
zipfile.write(path, arcname)
directories = ['lib', 'biglib', 'module']
if add_python:
directories.extend(['python', 'cfipython'])
if add_external:
directories.append('external')
if add_examples:
directories.append('src/UHH2/examples/config')
for directory in directories:
full_path = os.path.join(CMSSW_BASE, directory)
logger.debug("Checking directory %s" % full_path)
if os.path.exists(full_path):
logger.debug("Adding directory %s to tarball" % full_path)
check_directory(full_path)
tarfile.add(full_path,
os.path.join(CMSSW_VERSION, directory),
recursive=True)
# Note that data_dirs are only looked-for and added under the src/ folder.
# /data/ subdirs contain data files needed by the code
# /interface/ subdirs contain C++ header files needed e.g. by ROOT6
# /include/ is necessary for C++ headers for UHH2 objects
data_dirs = ['data', 'interface', 'include']
# Search for and tar up "data" directories in src/
src_path = os.path.join(CMSSW_BASE, 'src')
for root, _, _ in os.walk(src_path):
if os.path.basename(root) in data_dirs:
directory = root.replace(CMSSW_BASE, CMSSW_VERSION)
logger.debug("Adding data directory %s to tarball" % root)
check_directory(root)
