def check_published_files(project_id, revision=None, selected_files=None): #get list of files that should be in the publication es_client = new_es_client() publication = BaseESPublication(project_id=project_id, revision=revision, using=es_client) if selected_files: #it's type other, use this for comparison filepaths = selected_files else: filepaths = publication.related_file_paths() #empty dirs missing_files = [] existing_files = [] empty_folders = [] #strip leading forward slash from file paths updated_filepaths = [ file_path.strip('/') for file_path in filepaths if (file_path != '.Trash') ] pub_directory = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) if revision: pub_directory += 'v{}'.format(revision) #navigate through publication files paths and #compare to the previous list of files for pub_file in updated_filepaths: file_to_check = os.path.join(pub_directory, pub_file) try: if os.path.isfile(file_to_check): existing_files.append(pub_file) elif os.path.isdir(file_to_check): #check directory for items in it dir_list = os.listdir(file_to_check) if dir_list != []: existing_files.append(pub_file) else: empty_folders.append(pub_file) else: missing_files.append(pub_file) except OSError as exc: logger.info(exc) #send email if there are files/folders missing/empty if (missing_files or empty_folders): #log for potential later queries logger.info("check_published_files missing files: " + project_id + " " + str(missing_files)) logger.info("check_published_files empty folders: " + project_id + " " + str(empty_folders)) #send email to dev admins service = get_service_account_client() prj_admins = settings.DEV_PROJECT_ADMINS_EMAIL for admin in prj_admins: email_body = """ <p>Hello,</p> <p> The following project has been published with either missing files/folders or empty folders: <br> <b>{prjID} - revision {revision}</b> <br> Path to publication files: {pubFiles} </p> <p> These are the missing files/folders for this publication: <br> {missingFiles} </p> <p> These are the empty folders for this publication: <br> {emptyFolders} </p> This is a programmatically generated message. Do NOT reply to this message. """.format(pubFiles=pub_directory, prjID=project_id, missingFiles=missing_files, emptyFolders=empty_folders, revision=revision) send_mail( "DesignSafe Alert: Published Project has missing files/folders", email_body, settings.DEFAULT_FROM_EMAIL, [admin], html_message=email_body)
def copy_publication_files_to_corral(self, project_id): # Only copy published files while in prod if getattr(settings, 'DESIGNSAFE_ENVIRONMENT', 'dev') != 'default': return from designsafe.libs.elasticsearch.docs.publications import BaseESPublication import shutil publication = BaseESPublication(project_id=project_id) filepaths = publication.related_file_paths() if not len(filepaths): res = get_service_account_client().files.list( systemId='project-{project_uuid}'.format( project_uuid=publication.project.uuid), filePath='/') filepaths = [ _file.path.strip('/') for _file in res if (_file.name != '.' and _file.name != 'Trash') ] filepaths = list(set(filepaths)) filepaths = sorted(filepaths) base_path = ''.join(['/', publication.projectId]) os.chmod('/corral-repl/tacc/NHERI/published', 0o755) prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) if not os.path.isdir(prefix_dest): os.mkdir(prefix_dest) prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format( publication.project['uuid']) for filepath in filepaths: local_src_path = '{}/{}'.format(prefix_src, filepath) local_dst_path = '{}/{}'.format(prefix_dest, filepath) logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path) if os.path.isdir(local_src_path): try: #os.mkdir(local_dst_path) if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) shutil.copytree(local_src_path, local_dst_path) for root, dirs, files in os.walk(local_dst_path): for d in dirs: os.chmod(os.path.join(root, d), 0o555) for f in files: os.chmod(os.path.join(root, f), 0o444) os.chmod(local_dst_path, 0o555) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) else: try: if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) for root, dirs, files in os.walk( os.path.dirname(local_dst_path)): for d in dirs: os.chmod(os.path.join(root, d), 0o555) for f in files: os.chmod(os.path.join(root, f), 0o444) shutil.copy(local_src_path, local_dst_path) os.chmod(local_dst_path, 0o444) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) os.chmod(prefix_dest, 0o555) os.chmod('/corral-repl/tacc/NHERI/published', 0o555) save_to_fedora.apply_async(args=[project_id]) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': 'designsafe.storage.published', 'filePath': '/' + project_id, 'recurse': True }, queue='indexing')
def copy_publication_files_to_corral(self, project_id, revision=None, selected_files=None): """ Takes a project ID and copies project files to a published directory. :param str project_id: Project ID :param int revision: The revision number of the publication :param list of selected_files strings: Only provided if project type == other. """ es_client = new_es_client() publication = BaseESPublication(project_id=project_id, revision=revision, using=es_client) filepaths = publication.related_file_paths() if not len(filepaths) and selected_files: # Project is "Other" so we just copy the selected files filepaths = [ file_path.strip('/') for file_path in selected_files if (file_path != '.Trash') ] filepaths = list(set(filepaths)) filepaths = sorted(filepaths) base_path = ''.join(['/', publication.projectId]) os.chmod('/corral-repl/tacc/NHERI/published', 0o755) prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) if revision: prefix_dest += 'v{}'.format(revision) if not os.path.isdir(prefix_dest): os.mkdir(prefix_dest) prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format( publication.project['uuid']) for filepath in filepaths: local_src_path = '{}/{}'.format(prefix_src, filepath) local_dst_path = '{}/{}'.format(prefix_dest, filepath) logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path) if os.path.isdir(local_src_path): try: #os.mkdir(local_dst_path) if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) shutil.copytree(local_src_path, local_dst_path) for root, dirs, files in os.walk(local_dst_path): for d in dirs: os.chmod(os.path.join(root, d), 0o555) for f in files: os.chmod(os.path.join(root, f), 0o444) os.chmod(local_dst_path, 0o555) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) else: try: if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) for root, dirs, files in os.walk( os.path.dirname(local_dst_path)): for d in dirs: os.chmod(os.path.join(root, d), 0o555) for f in files: os.chmod(os.path.join(root, f), 0o444) shutil.copy(local_src_path, local_dst_path) os.chmod(local_dst_path, 0o444) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) os.chmod(prefix_dest, 0o555) os.chmod('/corral-repl/tacc/NHERI/published', 0o555) save_to_fedora.apply_async(args=[project_id, revision]) index_path = '/' + project_id if revision: index_path += 'v{}'.format(revision) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': 'designsafe.storage.published', 'filePath': index_path, 'recurse': True }, queue='indexing')