def list_outputs(self):
path = 's3:/{}/'.format(self.bucket)
result = rclone.with_config(self.config).run_cmd('lsd', [path])
if result['code'] != 0:
raise ValueError('failed to list output bucket: {}'.format(result))
out = result['out'].decode('utf8')
dirs = []
for line in out.splitlines():
name = line.split()[-1]
dirs.append(name)
return dirs
def test_copy_lsjson_and_delete(self):
source = "local:" + os.getcwd() + "/README.md"
with tempfile.TemporaryDirectory() as dest:
# copy
result = rclone.with_config(self.cfg).copy(source, "local:" + dest)
self.assertEqual(result.get('code'), 0)
self.assertEqual(result.get('out'), b'')
# lsjson
result = rclone.with_config(self.cfg).lsjson("local:" + dest)
self.assertEqual(result.get('code'), 0)
result_json = json.loads(result.get('out').decode("utf-8"))
self.assertGreater(len(result_json), 0)
self.assertEqual(result_json[0].get('Path'), 'README.md')
self.assertFalse(result_json[0].get('IsDir'))
# delete
result = rclone.with_config(self.cfg).delete("local:" + dest +
"/README.md")
self.assertEqual(result.get('code'), 0)
# lsjson to check that the file is gone
result = rclone.with_config(self.cfg).lsjson("local:" + dest)
self.assertEqual(result.get('code'), 0)
result_json = json.loads(result.get('out').decode("utf-8"))
self.assertEqual(len(result_json), 0)
def sync(source: Path = source_dir, dest: str = OSS_DEST):
import rclone
with open('rclone.conf') as cf:
cfg = cf.read()
cfg = cfg.format(
access_key_id=access_key_id,
secret_access_key=secret_access_key,
endpoint=endpoint,
provider=provider,
name=name,
type=_type
)
logging.info(f'syncing {source.absolute()} {dest}')
rst = rclone.with_config(cfg).sync(source=str(source.absolute()), dest=dest)
logging.info(rst)
def store_all(self, src_dir, ignore=[]):
"""
:param src_dir: path to local dir containing multiple test run dirs, e.g. ./output
:param ignore: list of subdirectories to ignore
"""
for f in os.listdir(src_dir):
if f in ignore:
continue
src = os.path.join(src_dir, f)
dest = 's3:/{}/{}'.format(self.bucket, f)
print('syncing {} to {}'.format(src, dest))
result = rclone.with_config(self.config).sync(src, dest)
if result['code'] != 0:
print('error storing {}: {}'.format(f, result['error']),
file=sys.stderr)
def move_to_drive_loop():
while True:
for img_name in db.keys():
img_data = db[img_name]
oldness = (datetime.now() - img_data['accessed_at']).seconds
if img_data['location'] == 'cached':
if oldness > MAX_SECONDS_BEFORE_DRIVE:
if os.path.isfile(f'{IMAGES_PATH}/{img_name}'):
os.remove(f'{IMAGES_PATH}/{img_name}')
db[img_name] = {'location': 'drive', 'accessed_at': img_data['accessed_at']}
elif img_data['location'] == 'local':
if not os.path.isfile(f'{IMAGES_PATH}/{img_name}'):
del(db[img_name])
else:
if oldness > MAX_SECONDS_BEFORE_DRIVE:
res = rclone.with_config(RCLONE_CONFIG).run_cmd(command="copy", extra_args=[f'local:{IMAGES_PATH}/{img_name}', f'{RCLONE_DRIVE_NAME}:{DRIVE_IMAGES_PATH}'])
if res['code'] == 0:
db[img_name] = {'location': 'drive', 'accessed_at': img_data['accessed_at']}
os.remove(f'{IMAGES_PATH}/{img_name}')
time.sleep(10)
def drive_backup(self, drive_dir=None):
self.append_log(self.log, '--- Backup to Google Drive')
self.backup_db()
tarname = self.compress_provision_dir('/home/kusanagi/')
cfg_file = '/root/.config/rclone/rclone.conf'
with open(cfg_file, 'rt') as f:
cfg = f.read()
# rc_options = ['--buffer-size=64M', '--transfers=5', '--drive-chunk-size=16M', '--drive-upload-cutoff=16M',
# '--log-file=%s' % self.log]
rc_options = ['--buffer-size=64M', '--log-file=%s' % self.log]
result = rclone.with_config(cfg).copy('%s.tar.gz' % tarname,
'GGD1:%s' % drive_dir,
rc_options)
res = result.get('code')
if int(res) == 0:
self.update_backup_record(2, 1)
# return {'status': 1, 'msq': 'Backup completed successfully'}
else:
self.update_backup_record(2, 0)
# return {'status': 0, 'msg': 'Check %s for more details' % self.log}
os.remove('%s.tar.gz' % tarname)
# Run one at a time
files = fs.ls('hrrr/sfc/' + str(x))[-3:]
#files2 = fs.ls('hrrr/prs/' + str(x))[-3:]
# Catch in case counter tries to pull non-existant day
except FileNotFoundError:
continue
# Import sfc files
for file in files:
#for file in files2:
item = str(file)
lst = item.split("/")
idx = len(lst) - 1
name = lst[idx]
path = "Staging/sfc/" + dest + "/" + name
#path = "Staging/prs/" + dest + "/" + name
fs.download(file, path)
# Upload files from local dir using rclone
print("Uploading to AWS...")
base = os.getcwd()
result = rclone.with_config(cfg).run_cmd(
command="sync",
extra_args=[base + "/Staging/sfc", "AWS test:transferfrompando/sfc"])
#result = rclone.with_config(cfg).run_cmd(command="sync", extra_args=[base + "/Staging/prs", "AWS test:transferfrompando/prs"])
# Change dir back to working directory
os.chdir("..")
# Delete medium directory upon completion
shutil.rmtree("Staging")
def _ensure_rclone_exists(self):
result = rclone.with_config(self.config).listremotes()
if result['code'] == -20:
raise EnvironmentError(
"the 'rclone' command must be present on the $PATH")
import rclone cfg_path = r'/Users/noelnathanieljimenofonseca/.config/rclone/rclone.conf' with open(cfg_path) as f: cfg = f.read() result = rclone.with_config(cfg).listremotes() #print(result) #COPY Google drive to S3 rclone.with_config(cfg).run_cmd(command='copy', extra_args=["-v", "--ignore-checksum", "mygoogledrive1:/", "mys3_1:/noelbeerbucket"]) #SYNC Google drive to S3 # rclone.with_config(cfg).run_cmd(command='sync', extra_args=["-v", "--ignore-checksum", "mygoogledrive1:/", "mys3_1:/noelbeerbucket"]) # S3 to Google Drive # rclone.with_config(cfg).run_cmd(command='sync', extra_args=["-v", "--ignore-checksum", "mys3_1:/noelbeerbucket", "mygoogledrive1:/"]) # print(rclone.with_config(cfg).run_cmd(command='ls', extra_args=["-v", "mys3_1:/noelbeerbucket"]))
if alarm_active():
if not alarm_logged:
logging.info('Alarm Active!')
alarm_logged = 1
events = i.event_gen(yield_nones=False, timeout_s=1)
events = list(events)
if events:
for event in events:
(_, type_names, path, filename) = event
if 'IN_CLOSE_WRITE' in type_names:
source = path + '/' + filename
dest = rclone_remote + path
logging.debug('Uploading: {}'.format(source))
logging.debug(' to {}'.format(dest))
result = rclone.with_config(rconfig).copy(source, dest)
if result.get('error'):
logging.error('OUTPUT: {}'.format(
result.get('out')))
logging.error('CODE: {}'.format(
result.get('code')))
logging.error('ERROR: {}'.format(
result.get('error')))
else:
uploaded_files += 1
else:
if alarm_logged:
logging.info(
"Alarm resolved, uploaded {} files".format(uploaded_files))
alarm_logged = 0
uploaded_files = 0
def test_execute_with_correct_command(self):
result = rclone.with_config(self.cfg)._execute(["echo", "123"])
self.assertEqual(result.get('code'), 0)
self.assertIsNotNone(result.get('out'))
import os
import rclone
from os.path import expanduser
import logging
# Enabled verbose logging
logging.basicConfig(level=logging.DEBUG,
format="%(asctime)s %(name)s [%(levelname)s]: %(message)s")
#cfg = """[local]
#type = local
#nounc = true"""
with open(expanduser('./rclone.conf')) as config_file:
config = config_file.read()
# List all top level folders with run_cmd
#result = rclone.with_config(config).run_cmd(command="lsd", extra_args=["remote:"])
# List top level folders using ls
#result = rclone.with_config(config).ls("remote:")
# Copy README.md to root to test
source = "./README.md"
result = rclone.with_config(config).copy(source, "remote:backup-test")
print(result.get('out'))
# Perform sync
import rclone
cfg_path = r'/home/nfonseca/.config/rclone/rclone.conf'
with open(cfg_path) as f:
cfg = f.read()
result = rclone.with_config(cfg).listremotes()
#print(result)
#rclone.with_config(cfg).run_cmd(command='sync', extra_args=["-v", "--ignore-checksum", "mygoogledrive2:/", "mys3:/noel-fonseca-bucket-1"])
print(
rclone.with_config(cfg).run_cmd(
command='ls', extra_args=["-v", "mys3:/noel-fonseca-bucket-1"]))
def test_listremoted(self):
result = rclone.with_config(self.cfg).listremotes()
self.assertEqual(result.get('code'), 0)
self.assertEqual(result.get('out'), b'local:\n')
def upload(local_path: str, remote_uri: str):
"Upload local to remote using rclone"
remote_uri = _to_rclone(remote_uri)
rclone.with_config(_load_rclone_cfg()).copy(local_path, remote_uri, flags=["-P"])
def fetch_image_from_drive(img_name):
if RCLONE_CONFIG is None:
return False
result = rclone.with_config(RCLONE_CONFIG).run_cmd(command="copy", extra_args=[f'{RCLONE_DRIVE_NAME}:{DRIVE_IMAGES_PATH}/{img_name}', f'local:{IMAGES_PATH}'])
return result['code'] == 0
def download(remote_uri: str, local_path: str):
"Download remote to local using rclone"
remote_uri = _to_rclone(remote_uri)
rclone.with_config(_load_rclone_cfg()).copy(remote_uri, local_path, flags=["-P"])
def test_execute_with_wrong_command(self):
result = rclone.with_config(self.cfg)._execute(
["command_not_valid", "some", "args"])
self.assertEqual(result.get('code'), -20)
self.assertIsInstance(result.get('error'), FileNotFoundError)
