def get_server_file_dict(server_id, server_directory):
"""
Returns an array of dictionaries which contain the file and directory listing
"""
db_session = DBSession()
server_file_dict = []
reachable = False
server = db_session.query(Server).filter(Server.id == server_id).first()
if server is not None:
server_impl = get_server_impl(server)
if server_impl is not None:
server_file_dict, reachable = server_impl.get_file_and_directory_dict(server_directory)
return server_file_dict, reachable
def get_server_file_dict(server_id, server_directory):
"""
Returns an array of dictionaries which contain the file and directory listing
"""
db_session = DBSession()
server_file_dict = []
reachable = False
server = db_session.query(Server).filter(Server.id == server_id).first()
if server is not None:
server_impl = get_server_impl(server)
if server_impl is not None:
server_file_dict, reachable = server_impl.get_file_and_directory_dict(
server_directory)
return server_file_dict, reachable
def check_server_reachability():
if not can_check_reachability(current_user):
abort(401)
hostname = request.args.get('hostname')
server_type = request.args.get('server_type')
server_url = request.args.get('server_url')
username = request.args.get('username')
password = request.args.get('password')
server_directory = request.args.get('server_directory')
server = Server(hostname=hostname,
server_type=server_type,
server_url=server_url,
username=username,
password=password,
server_directory=server_directory)
# The form is in the edit mode and the user clicks Validate Reachability
# If there is no password specified, try get it from the database.
if (server_type == ServerType.FTP_SERVER
or server_type == ServerType.SFTP_SERVER
or server_type == ServerType.SCP_SERVER) and password == '':
db_session = DBSession()
server_in_db = get_server(db_session, hostname)
if server_in_db is not None:
server.password = server_in_db.password
server_impl = get_server_impl(server)
is_reachable, error = server_impl.check_reachability()
if is_reachable:
return jsonify({'status': 'OK'})
else:
return jsonify({'status': error})
def start(self, db_session, logger, process_name):
self.db_session = db_session
try:
self.create_tar_job = self.db_session.query(CreateTarJob).filter(CreateTarJob.id == self.job_id).first()
if self.create_tar_job is None:
logger.error('Unable to retrieve create tar job: %s' % self.job_id)
return
self.create_tar_job.set_status(JobStatus.PROCESSING)
server_id = self.create_tar_job.server_id
server_directory = self.create_tar_job.server_directory
source_tars = self.create_tar_job.source_tars
contents = self.create_tar_job.contents
additional_packages = self.create_tar_job.additional_packages
new_tar_name = self.create_tar_job.new_tar_name
created_by = self.create_tar_job.created_by
date_string = datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S")
repo_dir = get_repository_directory()
temp_path = get_temp_directory() + str(date_string)
new_tar_path = os.path.join(temp_path, str(date_string))
try:
if not os.path.exists(temp_path):
self.create_tar_job.set_status('Creating temporary directories.')
self.db_session.commit()
os.makedirs(temp_path)
os.makedirs(new_tar_path, 7777)
# Untar source tars into the temp/timestamp directory
if source_tars:
self.create_tar_job.set_status('Extracting from source tar files.')
self.db_session.commit()
for source in source_tars.split(','):
with tarfile.open(os.path.join(repo_dir, source)) as tar:
tar.extractall(temp_path)
# Copy the selected contents from the temp/timestamp directory
# to the new tar directory
if contents:
self.create_tar_job.set_status('Copying selected tar contents.')
self.db_session.commit()
for f in contents.strip().split(','):
_, filename = os.path.split(f)
shutil.copy2(os.path.join(temp_path, filename), new_tar_path)
# Copy the selected additional packages from the repository to the new tar directory
if additional_packages:
self.create_tar_job.set_status('Copying selected additional files.')
for pkg in additional_packages.split(','):
self.db_session.commit()
shutil.copy2(os.path.join(repo_dir, pkg), new_tar_path)
self.create_tar_job.set_status('Tarring new file.')
self.db_session.commit()
tarname = os.path.join(temp_path, new_tar_name)
shutil.make_archive(tarname, format='tar', root_dir=new_tar_path)
make_file_writable(os.path.join(new_tar_path, tarname) + '.tar')
server = self.db_session.query(Server).filter(Server.id == server_id).first()
if server is not None:
self.create_tar_job.set_status('Uploading to external repository.')
self.db_session.commit()
server_impl = get_server_impl(server)
# If the new file already exists on the remote host, delete it
if new_tar_name in server_impl.get_file_list():
server_impl.delete_file(new_tar_name)
statinfo = os.stat(tarname + '.tar')
self.new_tar_size = statinfo.st_size
self.chunk_list = self.get_chunks(self.new_tar_size, self.new_tar_size / 1048576)
if isinstance(server_impl, FTPServer):
server_impl.upload_file(tarname + '.tar', new_tar_name + ".tar", sub_directory=server_directory,
callback=self.ftp_progress_listener)
elif isinstance(server_impl, SFTPServer):
server_impl.upload_file(tarname + '.tar', new_tar_name + ".tar", sub_directory=server_directory,
callback=self.sftp_progress_listener)
else:
server_impl.upload_file(tarname + '.tar', new_tar_name + ".tar", sub_directory=server_directory)
shutil.rmtree(temp_path, onerror=self.handleRemoveReadonly)
self.create_tar_job.set_status(JobStatus.COMPLETED)
self.db_session.commit()
except Exception:
self.create_tar_job.set_status(JobStatus.FAILED)
self.db_session.commit()
logger.exception('Exception while creating %s requested by %s - job id = %s',
new_tar_name, created_by, self.job_id)
shutil.rmtree(temp_path, onerror=self.handleRemoveReadonly)
os.remove(temp_path + '.tar')
finally:
self.db_session.close()
def start(self, db_session, logger, process_name):
# Save the db_session reference for progress_listener
self.db_session = db_session
try:
self.download_job = db_session.query(DownloadJob).filter(DownloadJob.id == self.job_id).first()
if self.download_job is None:
logger.error('Unable to retrieve download job: %s' % self.job_id)
return
output_file_path = get_repository_directory() + self.download_job.cco_filename
# Only download if the image (tar file) is not in the downloads directory.
# And, the image is a good one.
if not self.is_tar_file_valid(output_file_path):
user_id = self.download_job.user_id
user = db_session.query(User).filter(User.id == user_id).first()
if user is None:
logger.error('Unable to retrieve user: %s' % user_id)
preferences = db_session.query(Preferences).filter(Preferences.user_id == user_id).first()
if preferences is None:
logger.error('Unable to retrieve user preferences: %s' % user_id)
self.download_job.set_status(JobStatus.PROCESSING)
db_session.commit()
bsd = BSDServiceHandler(username=preferences.cco_username, password=preferences.cco_password,
image_name=self.download_job.cco_filename, PID=self.download_job.pid,
MDF_ID=self.download_job.mdf_id,
software_type_ID=self.download_job.software_type_id)
self.download_job.set_status('Preparing to download from cisco.com.')
db_session.commit()
bsd.download(output_file_path, callback=self.progress_listener)
tarfile_file_list = untar(output_file_path, get_repository_directory())
else:
tarfile_file_list = get_tarfile_file_list(output_file_path)
# Now transfers to the server repository
self.download_job.set_status('Transferring file to server repository.')
db_session.commit()
server = db_session.query(Server).filter(Server.id == self.download_job.server_id).first()
if server is not None:
server_impl = get_server_impl(server)
for filename in tarfile_file_list:
server_impl.upload_file(get_repository_directory() + filename, filename,
sub_directory=self.download_job.server_directory)
self.archive_download_job(db_session, self.download_job, JobStatus.COMPLETED)
db_session.commit()
except Exception:
try:
logger.exception('DownloadManager hit exception - download job = %s', self.job_id)
self.archive_download_job(db_session, self.download_job, JobStatus.FAILED, traceback.format_exc())
db_session.commit()
except Exception:
logger.exception('DownloadManager hit exception - download job = %s', self.job_id)
finally:
db_session.close()
def start(self, db_session, logger, process_name):
# Save the db_session reference for progress_listener
self.db_session = db_session
try:
self.download_job = db_session.query(DownloadJob).filter(
DownloadJob.id == self.job_id).first()
if self.download_job is None:
logger.error('Unable to retrieve download job: %s' %
self.job_id)
return
output_file_path = get_repository_directory(
) + self.download_job.cco_filename
# Only download if the image (tar file) is not in the downloads directory.
# And, the image is a good one.
if not self.is_tar_file_valid(output_file_path):
user_id = self.download_job.user_id
user = db_session.query(User).filter(
User.id == user_id).first()
if user is None:
logger.error('Unable to retrieve user: %s' % user_id)
preferences = db_session.query(Preferences).filter(
Preferences.user_id == user_id).first()
if preferences is None:
logger.error('Unable to retrieve user preferences: %s' %
user_id)
self.download_job.set_status(JobStatus.PROCESSING)
db_session.commit()
bsd = BSDServiceHandler(
username=preferences.cco_username,
password=preferences.cco_password,
image_name=self.download_job.cco_filename,
PID=self.download_job.pid,
MDF_ID=self.download_job.mdf_id,
software_type_ID=self.download_job.software_type_id)
self.download_job.set_status(
'Preparing to download from cisco.com.')
db_session.commit()
bsd.download(output_file_path, callback=self.progress_listener)
tarfile_file_list = untar(output_file_path,
get_repository_directory())
else:
tarfile_file_list = get_tarfile_file_list(output_file_path)
# Now transfers to the server repository
self.download_job.set_status(
'Transferring file to server repository.')
db_session.commit()
server = db_session.query(Server).filter(
Server.id == self.download_job.server_id).first()
if server is not None:
server_impl = get_server_impl(server)
for filename in tarfile_file_list:
server_impl.upload_file(
get_repository_directory() + filename,
filename,
sub_directory=self.download_job.server_directory)
self.archive_download_job(db_session, self.download_job,
JobStatus.COMPLETED)
db_session.commit()
except Exception:
try:
logger.exception(
'DownloadManager hit exception - download job = %s',
self.job_id)
self.archive_download_job(db_session, self.download_job,
JobStatus.FAILED,
traceback.format_exc())
db_session.commit()
except Exception:
logger.exception(
'DownloadManager hit exception - download job = %s',
self.job_id)
finally:
db_session.close()
def start(self, db_session, logger, process_name):
self.db_session = db_session
try:
self.create_tar_job = self.db_session.query(CreateTarJob).filter(
CreateTarJob.id == self.job_id).first()
if self.create_tar_job is None:
logger.error('Unable to retrieve create tar job: %s' %
self.job_id)
return
self.create_tar_job.set_status(JobStatus.PROCESSING)
server_id = self.create_tar_job.server_id
server_directory = self.create_tar_job.server_directory
source_tars = self.create_tar_job.source_tars
contents = self.create_tar_job.contents
additional_packages = self.create_tar_job.additional_packages
new_tar_name = self.create_tar_job.new_tar_name
created_by = self.create_tar_job.created_by
date_string = datetime.datetime.utcnow().strftime(
"%Y_%m_%d_%H_%M_%S")
repo_dir = get_repository_directory()
temp_path = get_temp_directory() + str(date_string)
new_tar_path = os.path.join(temp_path, str(date_string))
try:
if not os.path.exists(temp_path):
self.create_tar_job.set_status(
'Creating temporary directories.')
self.db_session.commit()
os.makedirs(temp_path)
os.makedirs(new_tar_path, 7777)
# Untar source tars into the temp/timestamp directory
if source_tars:
self.create_tar_job.set_status(
'Extracting from source tar files.')
self.db_session.commit()
for source in source_tars.split(','):
with tarfile.open(os.path.join(repo_dir,
source)) as tar:
tar.extractall(temp_path)
# Copy the selected contents from the temp/timestamp directory
# to the new tar directory
if contents:
self.create_tar_job.set_status(
'Copying selected tar contents.')
self.db_session.commit()
for f in contents.strip().split(','):
_, filename = os.path.split(f)
shutil.copy2(os.path.join(temp_path, filename),
new_tar_path)
# Copy the selected additional packages from the repository to the new tar directory
if additional_packages:
self.create_tar_job.set_status(
'Copying selected additional files.')
for pkg in additional_packages.split(','):
self.db_session.commit()
shutil.copy2(os.path.join(repo_dir, pkg), new_tar_path)
self.create_tar_job.set_status('Tarring new file.')
self.db_session.commit()
tarname = os.path.join(temp_path, new_tar_name)
shutil.make_archive(tarname,
format='tar',
root_dir=new_tar_path)
make_file_writable(
os.path.join(new_tar_path, tarname) + '.tar')
server = self.db_session.query(Server).filter(
Server.id == server_id).first()
if server is not None:
self.create_tar_job.set_status(
'Uploading to external repository.')
self.db_session.commit()
server_impl = get_server_impl(server)
# If the new file already exists on the remote host, delete it
if new_tar_name in server_impl.get_file_list():
server_impl.delete_file(new_tar_name)
statinfo = os.stat(tarname + '.tar')
self.new_tar_size = statinfo.st_size
self.chunk_list = self.get_chunks(
self.new_tar_size, self.new_tar_size / 1048576)
if isinstance(server_impl, FTPServer):
server_impl.upload_file(
tarname + '.tar',
new_tar_name + ".tar",
sub_directory=server_directory,
callback=self.ftp_progress_listener)
elif isinstance(server_impl, SFTPServer):
server_impl.upload_file(
tarname + '.tar',
new_tar_name + ".tar",
sub_directory=server_directory,
callback=self.sftp_progress_listener)
else:
server_impl.upload_file(tarname + '.tar',
new_tar_name + ".tar",
sub_directory=server_directory)
shutil.rmtree(temp_path, onerror=self.handleRemoveReadonly)
self.create_tar_job.set_status(JobStatus.COMPLETED)
self.db_session.commit()
except Exception:
self.create_tar_job.set_status(JobStatus.FAILED)
self.db_session.commit()
logger.exception(
'Exception while creating %s requested by %s - job id = %s',
new_tar_name, created_by, self.job_id)
shutil.rmtree(temp_path, onerror=self.handleRemoveReadonly)
os.remove(temp_path + '.tar')
finally:
self.db_session.close()
