def run(self):
def copy():
with self.input().open('r') as infile:
for line in infile:
yield line
logger.info('Copying %s to %s ...' % (self.src, self.dst))
t = time.time()
with self.output().open('w') as outfile:
for line in copy():
outfile.write(line)
t = round(time.time() - t, 2)
size = round(os.path.getsize(self.src) / 1024 * 1.0, 2) # in KB
kb_per_sec = round(size / t, 2)
logger.info('Done. Copied %s KB in %s seconds (%s KB/s)' % (size, t, kb_per_sec))
def run(self):
from saisoku import ThreadedCopy
import os
ThreadedCopy(src=self.src, dst=self.dst, threads=self.threads, filelist=self.filelist,
symlinks=self.symlinks, ignore=self.ignore, copymeta=self.copymeta, package=True)
if self.cleanup:
logger.info('Cleaning up %s...' % self.src)
with self.input()[0].open() as f:
filename = f.read().rstrip('\n')
logger.debug(' Removing %s...' % filename)
os.remove(filename) # delete tar.gz file in src
logger.debug(' Removing %s...' % self.filelist)
os.remove(self.filelist) # delete file list
def run(self):
"""
examples:
* s3://bucket/foo/bar.txt
* s3://bucket/foo/bar.txt?aws_access_key_id=xxx&aws_secret_access_key=yyy
"""
def copy():
with self.input().open('r') as infile:
for line in infile:
yield line
logger.info('Copying %s to %s ...' % (self.src, self.dst))
t = time.time()
with self.output().open('w') as outfile:
for line in copy():
outfile.write(line)
t = round(time.time() - t, 2)
size = round(os.path.getsize(self.dst) / 1024 * 1.0, 2) # in KB
kb_per_sec = round(size / t, 2)
logger.info('Done. Copied %s KB in %s seconds (%s KB/s)' % (size, t, kb_per_sec))
def run(self):
import tarfile
from scandir import scandir
import os
archives = []
src_path = os.path.abspath(self.src)
# create tar.gz of all files in directory
archive_name = '{}_saisoku_archive.tar.gz'.format(os.path.basename(src_path))
logger.info('Compressing files to %s...' % archive_name)
tar = tarfile.open(archive_name, "w:gz")
for item in scandir(self.src):
if item.is_file():
logger.debug(' Adding %s...' % item.name)
tar.add(item.path, item.name)
tar.close()
archives.append(archive_name)
with self.output().open('w') as f:
for archive in archives:
f.write('{archive}\n'.format(archive=archive))