def gen_rclone_cfg(args, filepath):
sa_files = glob.glob(os.path.join(args.service_account_dir, '*.json'))
if len(sa_files) == 0:
log('No json files found in ./{}'.format(args.service_account_dir),
'ERROR', args)
sys.exit(-1)
source_remote = None
dest_remote = None
src_is_crypt = False
dst_is_crypt = False
is_config_file_specified = False
parsed_config = None
if args.rclone_config_path:
is_config_file_specified = True
parsed_config = config_parser.parse_config(args.rclone_config_path,
args)
# Source parsing
if args.source:
source_remote, src_is_crypt = gen_remote_template(
args.source, parsed_config, args, is_config_file_specified)
# Destination parsing
if args.destination:
dest_remote, dst_is_crypt = gen_remote_template(
args.destination, parsed_config, args, is_config_file_specified)
with open(filepath, 'w') as fp:
for i, filename in enumerate(sa_files):
dir_path = os.path.dirname(Path(os.path.realpath(__file__)).parent)
filename = os.path.join(dir_path, filename)
filename = filename.replace(os.sep, '/')
index = i + 1
if source_remote:
if src_is_crypt:
remote_type = 'src'
fp.write(
source_remote.format(remote_type, index, filename,
remote_type, index, remote_type,
index))
else:
fp.write(source_remote.format('src', index, filename))
if dest_remote:
if dst_is_crypt:
remote_type = 'dst'
fp.write(
dest_remote.format(remote_type, index, filename,
remote_type, index, remote_type,
index))
else:
fp.write(dest_remote.format('dst', index, filename))
return i, src_is_crypt, dst_is_crypt
def parse_config(file_path, args):
try:
file = open(file_path, 'r')
except FileNotFoundError:
print("Rclone config file not found!")
sys.exit(-1)
config_content = file.read()
remotes_unparsed = []
remotes_tmp = config_content.split('[')
for i in range(1, len(remotes_tmp)):
remote_tmp = remotes_tmp[i].split(']\n')
# ['Remote Name', 'Remote Data']
remotes_unparsed.append([remote_tmp[0], remote_tmp[1]])
remotes_parsed = []
for remote in remotes_unparsed:
name = remote[0]
data = remote[1]
properties = []
data_tmp = data.split('\n')
# Remove empty array items caused by \n characters
data_tmp = list(filter(None, data_tmp))
for data in data_tmp:
data_split = data.split('=')
# ['Property', 'Value']
properties.append([data_split[0].strip(), data_split[1].strip()])
remotes_parsed.append([name, properties])
remotes = []
for remote in remotes_parsed:
name = remote[0]
properties = remote[1]
remote_type = None
team_drive = None
root_folder_id = None
remote = None
filename_encryption = None
directory_name_encryption = None
password = None
password2 = None
willSkipRemote = False
for prop in properties:
if prop[0] == "type":
remote_type = prop[1]
elif prop[0] == "team_drive":
team_drive = prop[1]
elif prop[0] == "root_folder_id":
root_folder_id = prop[1]
elif prop[0] == "remote":
remote = prop[1]
elif prop[0] == "filename_encryption":
filename_encryption = prop[1]
elif prop[0] == "directory_name_encryption":
directory_name_encryption = prop[1]
elif prop[0] == "password":
password = prop[1]
elif prop[0] == "password2":
password2 = prop[1]
if remote_type == "drive":
if team_drive or root_folder_id:
new_remote = drive_remote(name, team_drive, root_folder_id)
else:
willSkipRemote = True
helpers.log(
"Rclone remote '{}' is not available for use with service accounts please specify either a root_folder_id or team_drive."
.format(name), "WARN", args)
elif remote_type == "crypt":
new_remote = crypt_remote(name, remote, filename_encryption,
directory_name_encryption, password,
password2)
if not willSkipRemote:
remotes.append(new_remote)
return remotes
def main():
# Sets the scripts SIGINT handler to our exit_handler
signal(SIGINT, exit_handler)
# Check if RClone is installed, if it isn't, exit
ret = helpers.check_rclone_exists()
# Parse args
args = arg_parser.parse_args()
# Log that rclone was detected
helpers.log('RClone detected: {}'.format(ret), 'INFO', args)
# Generate config
rclone_generated_config_path = args.generated_config_path
source_path = ''
# Use either source remote or source path, if neither exist exit
if args.source:
source_path = args.source
elif args.source_path:
source_path = args.source_path
else:
helpers.log(
'A source is required, please use either --source or --source_path.',
'ERROR', args)
sys.exit(-1)
# If both a remote and a path exist combine them using RClone syntax
if args.source and args.source_path:
source_path += ":" + args.source_path
# See comments above
destination_path = ''
if args.destination:
destination_path = args.destination
elif args.destination_path:
destination_path = args.destination_path
else:
helpers.log(
'A destination is required, please use either --destination or --destination_path.',
'ERROR', args)
sys.exit(-1)
if args.destination and args.destination_path:
destination_path += ":" + args.destination_path
# Set id initially to the starting SA id
id = args.sa_start_id
end_id = args.sa_end_id
helpers.log('Generating RClone config file...', 'INFO', args)
# Generate RClone config file
end_id, src_is_crypt, dst_is_crypt = config_gen.gen_rclone_cfg(
args, rclone_generated_config_path)
time_start = time.time()
helpers.log(
'Starting job: {}, at {}'.format(args.name, time.strftime('%H:%M:%S')),
'INFO', args)
helpers.log('Source: ' + source_path, 'INFO', args)
helpers.log('Destination: ' + destination_path, 'INFO', args)
helpers.log('AutoRClone Log: ' + args.log_file, 'INFO', args)
helpers.log('RClone Log: ' + args.rclone_log_file, 'INFO', args)
helpers.log('Calculating source size, please wait', 'INFO', args)
# Initialise exit counter outside of loop so it keeps it's value
exit_counter = 0
error_counter = 0
global_bytes_transferred = 0
while id <= end_id + 1:
if id == end_id + 1:
break
# Construct destination and source labels
src_label = 'src' + '{0:03d}'.format(id) + ':'
dst_label = 'dst' + '{0:03d}'.format(id) + ':'
if src_is_crypt:
src_label = 'src' + '{0:03d}_crypt'.format(id) + ':'
if dst_is_crypt:
dst_label = 'dst' + '{0:03d}_crypt'.format(id) + ':'
# Fix for local paths that do not use a remote
if args.source_path:
if not args.source:
src_label = args.source_path
else:
src_label += args.source_path
if args.destination_path:
if not args.destination:
dst_label = args.destination_path
else:
dst_label += args.destination_path
if id == args.sa_start_id:
amount_to_transfer_bytes = helpers.calculate_path_size(
src_label, rclone_generated_config_path)
amount_to_transfer = helpers.convert_bytes_to_best_unit(
amount_to_transfer_bytes)
helpers.log('Source size: ' + amount_to_transfer + '\n', 'INFO',
args)
# Construct RClone command
rclone_cmd = 'rclone --config {} '.format(rclone_generated_config_path)
if args.copy:
rclone_cmd += 'copy '
elif args.move:
rclone_cmd += 'move '
elif args.sync:
rclone_cmd += 'sync '
else:
helpers.log(
'Please specify an operation (--copy, --move or --sync)',
'ERROR', args)
sys.exit()
rclone_cmd += '--drive-server-side-across-configs --drive-acknowledge-abuse --ignore-existing --rc '
rclone_cmd += '--rc-addr=\"localhost:{}\" --tpslimit {} --transfers {} --drive-chunk-size {} --bwlimit {} --log-file {} '.format(
args.port, args.tpslimit, args.transfers, args.drive_chunk_size,
args.bwlimit, args.rclone_log_file)
if args.dry_run:
rclone_cmd += '--dry-run '
if args.v:
rclone_cmd += '-v '
if args.vv:
rclone_cmd += '-vv '
if args.delete_empty_src_dirs:
rclone_cmd += '--delete-empty-src-dirs '
if args.create_empty_src_dirs:
rclone_cmd += '--create-empty-src-dirs '
# Add source and destination
rclone_cmd += '\"{}\" \"{}\"'.format(src_label, dst_label)
# If we're not on windows append ' &' otherwise append 'start /b ' to the start of rclone_cmd
if not helpers.is_windows():
rclone_cmd = rclone_cmd + " &"
else:
rclone_cmd = "start /b " + rclone_cmd
try:
subprocess.check_call(rclone_cmd, shell=True)
helpers.log('Executing RClone command: {}'.format(rclone_cmd),
'DEBUG', args)
time.sleep(10)
except subprocess.SubprocessError as error:
helpers.log('Error executing RClone command: {}'.format(error),
'ERROR', args)
sys.exit(-1)
# Counter for errors encountered when attempting to get RClone rc stats (per sa)
sa_error_counter = 0
# Counter that's incremented when no bytes are transferred over a time period
dead_transfer_counter = 0
# Updated on each loop
last_bytes_transferred = 0
# Counter for amount of successful stat retrievals from RClone rc (per sa)
sa_success_counter = 0
job_started = False
# Get RClone PID and store it
try:
response = subprocess.check_output(
'rclone rc --rc-addr="localhost:{}" core/pid'.format(
args.port),
shell=True,
stderr=subprocess.DEVNULL)
pid = json.loads(response.decode('utf-8').replace('\0', ''))['pid']
global PID
PID = int(pid)
except subprocess.SubprocessError as error:
pass
# Loop infinitely until loop is broken out of
while True:
# RClone rc stats command
rc_cmd = 'rclone rc --rc-addr="localhost:{}" core/stats'.format(
args.port)
try:
# Run command and store response
response = subprocess.check_output(rc_cmd,
shell=True,
stderr=subprocess.DEVNULL)
# Increment success counter
sa_success_counter += 1
# Reset error counter
sa_error_counter = 0
if job_started and sa_success_counter >= 9:
sa_error_counter = 0
sa_success_counter = 0
except subprocess.SubprocessError as error:
sa_error_counter += 1
error_counter = error_counter + 1
if sa_error_counter >= 3:
sa_success_counter = 0
if error_counter >= 9:
finish_job(args, time_start)
sys.exit(0)
helpers.log(
'Encountered 3 successive errors when trying to contact rclone, switching accounts ({}/3)'
.format(error_counter / sa_error_counter), 'INFO',
args)
break
continue
response_processed = response.decode('utf-8').replace('\0', '')
response_processed_json = json.loads(response_processed)
bytes_transferred = int(response_processed_json['bytes'])
checks_done = int(response_processed_json['checks'])
transfer_speed_bytes = (bytes_transferred -
last_bytes_transferred) / 4
# I'm using The International Engineering Community (IEC) Standard, eg. 1 GB = 1000 MB, if you think otherwise, fight me!
best_unit_transferred = helpers.convert_bytes_to_best_unit(
bytes_transferred)
transfer_speed = helpers.convert_bytes_to_best_unit(
transfer_speed_bytes)
#transfers = response_processed_json['transferring']
#for file in transfers:
# name = file['name']
# name_hashed = hashlib.sha1(bytes(name, encoding='utf8')).hexdigest()
# size_bytes = file['size']
# helpers.log('File: {} ({}) is {} bytes'.format(name, name_hashed, size_bytes), 'DEBUG', args)
# if not name_hashed in file_names:
# file_names.append(name_hashed)
# file_sizes.append(size_bytes)
#helpers.log("file_names = " + str(file_names), 'DEBUG', args)
#helpers.log("file_sizes = " + str(file_sizes), 'DEBUG', args)
#amount_to_transfer_bytes = sum(file_sizes)
#amount_to_transfer = helpers.convert_bytes_to_best_unit(amount_to_transfer_bytes)
bytes_left_to_transfer = int(
amount_to_transfer_bytes) - bytes_transferred
eta = helpers.calculate_transfer_eta(bytes_left_to_transfer,
transfer_speed_bytes)
helpers.log('{}/{} @ {}/s Files Checked: {} SA: {} ETA: {}'.format(
best_unit_transferred, amount_to_transfer, transfer_speed,
checks_done, id, eta) + (" " * 10),
"INFO",
args,
end='\r')
# continually no ...
if bytes_transferred - last_bytes_transferred == 0:
dead_transfer_counter += 1
helpers.log(
'No bytes transferred, RClone may be dead ({}/{})'.format(
dead_transfer_counter, TRANSFER_DEAD_THRESHOLD) +
(" " * 10), 'DEBUG', args)
else:
dead_transfer_counter = 0
job_started = True
last_bytes_transferred = bytes_transferred
# Stop by error (403, etc) info
if bytes_transferred >= MAX_TRANSFER_BYTES or dead_transfer_counter >= TRANSFER_DEAD_THRESHOLD:
if helpers.is_windows():
kill_cmd = 'taskkill /PID {} /F'.format(PID)
else:
kill_cmd = "kill -9 {}".format(PID)
try:
subprocess.check_call(kill_cmd, shell=True)
helpers.log(
'Transfer limit reached or RClone is not transferring any data, switching service accounts',
'INFO', args)
amount_to_transfer_bytes -= bytes_transferred
amount_to_transfer = helpers.convert_bytes_to_best_unit(
amount_to_transfer_bytes)
global_bytes_transferred += bytes_transferred
except:
pass
if dead_transfer_counter >= TRANSFER_DEAD_THRESHOLD:
try:
exit_counter += 1
except:
exit_counter = 1
else:
# clear cnt if there is one time
exit_counter = 0
# Regard continually exit as *all done*.
if exit_counter >= SA_EXIT_TRESHOLD:
# Exit directly rather than switch to next account.
finish_job(args, time_start)
sys.exit(0)
break
time.sleep(4)
id = id + 1
def finish_job(args, time_start):
helpers.log('Job FINISHED (this message will be better soon)', 'INFO',
args)
def gen_remote_template(src_or_dest, parsed_config, args, is_config_file_specified):
remote_template = None
found = False
remote_is_crypt = False
if is_config_file_specified:
for remote in parsed_config:
if remote.remote_name == src_or_dest:
found = True
if isinstance(remote, config_parser.crypt_remote):
crypt_remote_parts = remote.remote.split(':')
unencrypted_remote_found = False
remote_is_crypt = True
for unencrypted_remote in parsed_config:
if unencrypted_remote.remote_name == crypt_remote_parts[0]:
unencrypted_remote_found = True
remote_template = '[{}{:03d}]\n' \
'type = drive\n' \
'scope = drive\n' \
'service_account_file = {}\n'
if unencrypted_remote.team_drive:
remote_template += '{} = {}\n\n'.format('team_drive', unencrypted_remote.team_drive)
#elif unencrypted_remote.source_path_id:
# remote_template += '{} = {}\n\n'.format('source_path_id', unencrypted_remote.source_path_id)
if unencrypted_remote_found:
break
if not unencrypted_remote_found:
log('Invalid RClone config, crypt remote with remote that does not exist!', 'ERROR', args)
sys.exit(-1)
remote_template += '[{}{:03d}_crypt]\n' \
'type = crypt\n' \
'remote = {}{:03d}:' + crypt_remote_parts[1] + '\n' \
'filename_encryption = ' + remote.filename_encryption + '\n' \
'directory_name_encryption = ' + remote.directory_name_encryption + '\n' \
'password = '******'\n'
if remote.password2:
remote_template += 'password2 = ' + remote.password2 + '\n\n'
else:
remote_template += '\n'
else:
remote_template = "[{}{:03d}]\n" \
"type = drive\n" \
"scope = drive\n" \
"service_account_file = {}\n"
if remote.team_drive:
remote_template += "{} = {}\n\n".format("team_drive", remote.team_drive)
elif remote.source_path_id:
remote_template += "{} = {}\n\n".format("source_path_id", remote.source_path_id)
# If remote is found exit loop
if found:
break
if not found:
if len(src_or_dest) == 33:
folder_or_team_drive_src = 'root_folder_id'
elif len(src_or_dest) == 19:
folder_or_team_drive_src = 'team_drive'
elif is_config_file_specified:
log('The config file ' + args.rclone_config_path + ' was specified, ' + src_or_dest +
' was not a valid remote found in the config file, and is not a valid Team Drive ID or publicly shared Root Folder ID', "ERROR", args)
sys.exit(-1)
else:
log(src_or_dest + ' is not a valid Team Drive ID or publicly shared Root Folder ID', 'ERROR', args)
sys.exit(-1)
remote_template = "[{}{:03d}]\n" \
"type = drive\n" \
"scope = drive\n" \
"service_account_file = {}\n"
remote_template += "{} = {}\n\n".format(folder_or_team_drive_src, src_or_dest)
return remote_template, remote_is_crypt
