def move_rename(src, file_list, dest, user_pass):
try:
file_not_exists = []
existing_file_list = []
if src.endswith('/'):
src = src[:-1]
if dest.endswith('/'):
dest = dest[:-1]
for file in file_list:
file_to_move = src + '/%s' % file
dest_path = dest + '/%s' % file
if os.path.exists(dest_path):
existing_file_list.append(dest_path)
continue
if not os.path.exists(file_to_move):
file_not_exists.append(file_to_move)
else:
run_services.run_basic_services(
"echo %s | sudo -S mv %s %s" %
(user_pass, file_to_move, dest))
if file_not_exists:
return '{"success": 0, "msg": ["%s", file does not exists!!!]}' % file_not_exists
elif existing_file_list:
return '{"success": 0, "msg": ["%s", file already exists!!!]}' % existing_file_list
return '{"success": 1}'
except Exception as e:
log.error("Exception in copy_move_delete ==> move()")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
def extract_archive(file_name, root_folder, override, user_pass):
'''
Extracts .zip file
:return: Returns success 1 if successful or success 0 with error message
'''
log.info("\nExtracting archived file\n")
file_name = ast.literal_eval(file_name)
if not file_name:
log.error("File to extract is None")
return '{"success": 0, "msg": ["Empty File list"]}'
file_name = file_name[0]
try:
if '.zip' not in file_name:
log.error("%s is not a zip file" % file_name)
return '{"success": 0, "msg": ["%s not a zip file"]}' % file_name
if not os.path.exists(root_folder + file_name):
log.error("%s file does not exists!!!" % file_name)
return '{"success": 0, "msg": ["%s file does not exists!!!"]}' % file_name
if os.path.isfile(root_folder + file_name):
zip_folder_name = file_name.strip().split('.')[0]
if os.path.exists(root_folder + zip_folder_name):
if override == "true":
zip_file_path = root_folder + zip_folder_name
run_services.run_basic_services(
"echo %s | sudo -S rm -rf %s" %
(user_pass, zip_file_path))
else:
log.error("File already exists, set override to true!!!")
return '{"success": 0, "msg": "exists"}'
zip_file_path = root_folder + file_name
zipf = zipfile.ZipFile(zip_file_path)
zipf.extractall(root_folder + zip_folder_name)
zipf.close()
chown(root_folder + zip_folder_name, username, groupname,
user_pass)
else:
log.error("%s is not a file" % file_name)
return '{"success": 0, "msg": ["%s is not a file"]}' % file_name
except Exception as e:
log.error("Error while extracting an archived file")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
log.info("Successfully extracted archived file\n")
return '{"success": 1}'
def make_directory(folder_to_create, user_pass):
'''
Creates a new directory in remote server
:return: Returns success 1 if successful or success 0 with error message
'''
log.info("\nMaking a new Directory\n")
count = 0
inner_dict = {}
mkdir_cmd = "echo %s | sudo -S mkdir -p %s" % (user_pass, folder_to_create)
result = run_services.run_basic_services(mkdir_cmd)
if result != "success":
# count is set to 1 if any one file returns error
count = 1
success_value = result.strip().split(":")
inner_dict[folder_to_create] = success_value[-1].strip()
if count == 0:
log.info("\nSuccessfully created a new Directory\n")
chown(folder_to_create, username, groupname, user_pass)
return '{"success": 1}'
else:
log.error("\nError while creating a new Directory\n")
return '{"success": 0, "msg": [%s]}' % json.dumps(inner_dict)
def copy(src, file_list, dest, user_pass, *args):
log.info("\nCopying\n")
try:
file_not_exists = []
if src.endswith('/'):
src = src[:-1]
if dest.endswith('/'):
dest = dest[:-1]
if not os.path.exists(dest):
os.makedirs(dest)
for file in file_list:
file_to_copy = src + '/%s' % file
if not os.path.exists(file_to_copy):
file_not_exists.append(file_to_copy)
else:
if os.path.isdir(file_to_copy):
unique_id = str(time.time()).split('.')[0]
copied_file = dest + '/%s_%s' % (file, unique_id)
run_services.run_basic_services(
"echo %s | sudo -S cp -r %s %s" %
(user_pass, file_to_copy, copied_file))
chown(copied_file, username, groupname, user_pass)
else:
unique_id = str(time.time()).split('.')[0]
copied_file = dest + '/%s_%s' % (file, unique_id)
run_services.run_basic_services(
"echo %s | sudo -S cp %s %s" %
(user_pass, file_to_copy, copied_file))
chown(copied_file, username, groupname, user_pass)
if file_not_exists:
return '{"success": 0, "msg": ["%s", file does not exists!!!]}' % file_not_exists
return '{"success": 1}'
except Exception as e:
log.error("Exception in copy_move_delete ==> copy()")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
def delete(src, files, user_pass):
try:
file_not_exists = []
if src.endswith('/'):
src = src[:-1]
for file in files:
file_path = src + '/%s' % file
if not os.path.exists(file_path):
file_not_exists.append(file_path)
run_services.run_basic_services("echo %s | sudo -S rm -rf %s" %
(user_pass, file_path))
if file_not_exists:
return '{"success": 0, "msg": ["%s", file does not exists!!!]}' % file_not_exists
return '{"success": 1}'
except Exception as e:
log.error("Exception in copy_move_delete ==> move()")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
def create_archive(zip_folder_name, file_name_list, root_folder, override,
user_pass):
'''
Creates archive (.zip) of the given folder or files, sent from a post request.
:return: Returns success 1 if successful or success 0 with error message
'''
log.info("\nCreating an archive file\n")
if '.zip' in zip_folder_name:
zip_folder_name = zip_folder_name.replace('.zip', '')
zip_folder_name = zip_folder_name + '.zip'
if file_name_list:
try:
os.chdir(root_folder)
if os.path.exists(root_folder + zip_folder_name):
if override == "true":
os.remove(root_folder + zip_folder_name)
else:
return '{"success": 0, "msg": "exists"}'
lst = " ".join(str(x) for x in file_name_list)
archive_cmd = "echo %s | sudo -S zip %s %s" % (
user_pass, zip_folder_name, lst)
run_services.run_basic_services(archive_cmd)
chown(zip_folder_name, username, groupname, user_pass)
except Exception as e:
log.error("Exception while creating an archive file")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
else:
log.error("List of files and folders to archive is empty")
return '{"success": 0, "msg": ["Empty folder list"]}'
log.info("successfully created an archive file\n")
return '{"success": 1}'
def chown(file_path, user, group, user_pass):
try:
chown_cmd = "echo %s | sudo -S chown %s:%s %s" % (user_pass, user,
group, file_path)
result = run_services.run_basic_services(chown_cmd)
if result != "success":
return '{"success": 0, "msg": ["Error while changing the owner", %s]}' % result.strip(
)
else:
return '{"success": 1}'
except Exception as e:
log.error("Exception in chown_chmod.py ==> chown()")
log.error(e)
return '{"success": 0, "msg": ["%s"]}' % e
def hdfs_write():
'''
Writes to hdfs from local/remote server
:return: Returns success 1 if successful or success 0 with error message
'''
log.info("\nWriting to HDFS\n")
header_key = request.headers.get('API-KEY')
api_status = check_apiKey(header_key)
if api_status == 'success':
loaded_json = json.loads(request.data.decode())
root_folder = loaded_json['root_folder']
destination = loaded_json['destination']
file_name_list = loaded_json['file_name']
file_name_list = ast.literal_eval(file_name_list)
count = 0
inner_dict = {}
for file_name in file_name_list:
file_name = '%s%s' % (root_folder, file_name)
hdfs_write_cmd = '%shadoop fs -put %s %s' % (hadoop_bin, file_name,
destination)
result = run_services.run_basic_services(hdfs_write_cmd)
if result != "success":
# count is set to 1 if any one file returns error
count = 1
success_value = result.strip().split(":")
inner_dict[file_name] = success_value[-1].strip()
if count == 0:
log.info("\nSuccessfully written in HDFS\n")
return '{"success": 1}'
else:
log.error("\nError writing files in HDFS\n")
return '{"success": 0, "msg": [%s]}' % json.dumps(inner_dict)
else:
return api_status
