def check_published_files(project_id, revision=None, selected_files=None):
#get list of files that should be in the publication
es_client = new_es_client()
publication = BaseESPublication(project_id=project_id,
revision=revision,
using=es_client)
if selected_files:
#it's type other, use this for comparison
filepaths = selected_files
else:
filepaths = publication.related_file_paths()
#empty dirs
missing_files = []
existing_files = []
empty_folders = []
#strip leading forward slash from file paths
updated_filepaths = [
file_path.strip('/') for file_path in filepaths
if (file_path != '.Trash')
]
pub_directory = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if revision:
pub_directory += 'v{}'.format(revision)
#navigate through publication files paths and
#compare to the previous list of files
for pub_file in updated_filepaths:
file_to_check = os.path.join(pub_directory, pub_file)
try:
if os.path.isfile(file_to_check):
existing_files.append(pub_file)
elif os.path.isdir(file_to_check):
#check directory for items in it
dir_list = os.listdir(file_to_check)
if dir_list != []:
existing_files.append(pub_file)
else:
empty_folders.append(pub_file)
else:
missing_files.append(pub_file)
except OSError as exc:
logger.info(exc)
#send email if there are files/folders missing/empty
if (missing_files or empty_folders):
#log for potential later queries
logger.info("check_published_files missing files: " + project_id +
" " + str(missing_files))
logger.info("check_published_files empty folders: " + project_id +
" " + str(empty_folders))
#send email to dev admins
service = get_service_account_client()
prj_admins = settings.DEV_PROJECT_ADMINS_EMAIL
for admin in prj_admins:
email_body = """
<p>Hello,</p>
<p>
The following project has been published with either missing files/folders or empty folders:
<br>
<b>{prjID} - revision {revision}</b>
<br>
Path to publication files: {pubFiles}
</p>
<p>
These are the missing files/folders for this publication:
<br>
{missingFiles}
</p>
<p>
These are the empty folders for this publication:
<br>
{emptyFolders}
</p>
This is a programmatically generated message. Do NOT reply to this message.
""".format(pubFiles=pub_directory,
prjID=project_id,
missingFiles=missing_files,
emptyFolders=empty_folders,
revision=revision)
send_mail(
"DesignSafe Alert: Published Project has missing files/folders",
email_body,
settings.DEFAULT_FROM_EMAIL, [admin],
html_message=email_body)
def copy_publication_files_to_corral(self, project_id):
# Only copy published files while in prod
if getattr(settings, 'DESIGNSAFE_ENVIRONMENT', 'dev') != 'default':
return
from designsafe.libs.elasticsearch.docs.publications import BaseESPublication
import shutil
publication = BaseESPublication(project_id=project_id)
filepaths = publication.related_file_paths()
if not len(filepaths):
res = get_service_account_client().files.list(
systemId='project-{project_uuid}'.format(
project_uuid=publication.project.uuid),
filePath='/')
filepaths = [
_file.path.strip('/') for _file in res
if (_file.name != '.' and _file.name != 'Trash')
]
filepaths = list(set(filepaths))
filepaths = sorted(filepaths)
base_path = ''.join(['/', publication.projectId])
os.chmod('/corral-repl/tacc/NHERI/published', 0o755)
prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if not os.path.isdir(prefix_dest):
os.mkdir(prefix_dest)
prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(
publication.project['uuid'])
for filepath in filepaths:
local_src_path = '{}/{}'.format(prefix_src, filepath)
local_dst_path = '{}/{}'.format(prefix_dest, filepath)
logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path)
if os.path.isdir(local_src_path):
try:
#os.mkdir(local_dst_path)
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
shutil.copytree(local_src_path, local_dst_path)
for root, dirs, files in os.walk(local_dst_path):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
os.chmod(local_dst_path, 0o555)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
else:
try:
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
for root, dirs, files in os.walk(
os.path.dirname(local_dst_path)):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
shutil.copy(local_src_path, local_dst_path)
os.chmod(local_dst_path, 0o444)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
os.chmod(prefix_dest, 0o555)
os.chmod('/corral-repl/tacc/NHERI/published', 0o555)
save_to_fedora.apply_async(args=[project_id])
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': 'designsafe.storage.published',
'filePath': '/' + project_id,
'recurse': True
},
queue='indexing')
def copy_publication_files_to_corral(self,
project_id,
revision=None,
selected_files=None):
"""
Takes a project ID and copies project files to a published directory.
:param str project_id: Project ID
:param int revision: The revision number of the publication
:param list of selected_files strings: Only provided if project type == other.
"""
es_client = new_es_client()
publication = BaseESPublication(project_id=project_id,
revision=revision,
using=es_client)
filepaths = publication.related_file_paths()
if not len(filepaths) and selected_files:
# Project is "Other" so we just copy the selected files
filepaths = [
file_path.strip('/') for file_path in selected_files
if (file_path != '.Trash')
]
filepaths = list(set(filepaths))
filepaths = sorted(filepaths)
base_path = ''.join(['/', publication.projectId])
os.chmod('/corral-repl/tacc/NHERI/published', 0o755)
prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if revision:
prefix_dest += 'v{}'.format(revision)
if not os.path.isdir(prefix_dest):
os.mkdir(prefix_dest)
prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(
publication.project['uuid'])
for filepath in filepaths:
local_src_path = '{}/{}'.format(prefix_src, filepath)
local_dst_path = '{}/{}'.format(prefix_dest, filepath)
logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path)
if os.path.isdir(local_src_path):
try:
#os.mkdir(local_dst_path)
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
shutil.copytree(local_src_path, local_dst_path)
for root, dirs, files in os.walk(local_dst_path):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
os.chmod(local_dst_path, 0o555)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
else:
try:
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
for root, dirs, files in os.walk(
os.path.dirname(local_dst_path)):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
shutil.copy(local_src_path, local_dst_path)
os.chmod(local_dst_path, 0o444)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
os.chmod(prefix_dest, 0o555)
os.chmod('/corral-repl/tacc/NHERI/published', 0o555)
save_to_fedora.apply_async(args=[project_id, revision])
index_path = '/' + project_id
if revision:
index_path += 'v{}'.format(revision)
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': 'designsafe.storage.published',
'filePath': index_path,
'recurse': True
},
queue='indexing')
