def create_organization(auth_user, organization_name, organization_sfid):
"""
Creates a github organization and returns the newly created github organization in dict format.
:param auth_user: authorization for this user.
:type auth_user: AuthUser
:param organization_name: The github organization name.
:type organization_name: string
:param organization_sfid: The SFDC ID for the github organization.
:type organization_sfid: string/None
:return: dict representation of the new github organization object.
:rtype: dict
"""
# Validate user is authorized for this SFDC ID.
can_access = check_user_authorization(auth_user, organization_sfid)
if not can_access['valid']:
return can_access['errors']
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
except DoesNotExist:
cla.log.debug('creating organization: {} with sfid: {}'.format(organization_name, organization_sfid))
github_organization.set_organization_name(str(organization_name))
github_organization.set_organization_sfid(str(organization_sfid))
github_organization.set_project_sfid(str(organization_sfid))
github_organization.save()
return github_organization.to_dict()
cla.log.warning('organization already exists: {} - unable to create'.format(organization_name))
return {'errors': {'organization_name': 'This organization already exists'}}
def update_organization(organization_name, # pylint: disable=too-many-arguments
organization_sfid=None,
organization_installation_id=None):
"""
Updates a github organization and returns the newly updated org in dict format.
Values of None means the field will not be updated.
:param organization_name: The github organization name.
:type organization_name: string
:param organization_sfid: The SFDC identifier ID for the organization.
:type organization_sfid: string/None
:param organization_installation_id: The github app installation id.
:type organization_installation_id: string/None
:return: dict representation of the new github organization object.
:rtype: dict
"""
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
except DoesNotExist as err:
cla.log.warning('organization does not exist: {} - unable to update'.format(organization_name))
return {'errors': {'repository_id': str(err)}}
github_organization.set_organization_name(organization_name)
if organization_installation_id:
github_organization.set_organization_installation_id(organization_installation_id)
if organization_sfid:
github_organization.set_organization_sfid(organization_sfid)
github_organization.save()
cla.log.debug('updated organization: {}'.format(organization_name))
return github_organization.to_dict()
def get_organization_repositories(organization_name):
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
if github_organization.get_organization_installation_id() is not None:
cla.log.debug('GitHub Organization ID: {}'.format(github_organization.get_organization_installation_id()))
try:
installation = GitHubInstallation(github_organization.get_organization_installation_id())
except Exception as e:
msg = ('Unable to load repositories from organization: {} ({}) due to GitHub '
'installation permission problem or other issue, error: {} - returning error response'.
format(organization_name, github_organization.get_organization_installation_id(), e))
cla.log.warn(msg)
return {'errors': {'organization_name': organization_name, 'error': msg}}
if installation.repos:
repos = []
for repo in installation.repos:
repos.append(repo.full_name)
return repos
else:
cla.log.debug('No repositories found for Github installation id: {}'.
format(github_organization.get_organization_installation_id()))
return []
except DoesNotExist as err:
cla.log.warning('organization name {} does not exist, error: {}'.format(organization_name, err))
return {'errors': {'organization_name': organization_name, 'error': str(err)}}
def delete_organization(auth_user, organization_name):
"""
Deletes a github organization based on Name.
:param organization_name: The Name of the github organization.
:type organization_name: Name
"""
# Retrieve SFDC ID for this organization
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
except DoesNotExist as err:
cla.log.warning('organization does not exist: {} - unable to delete'.format(organization_name))
return {'errors': {'organization_name': str(err)}}
organization_sfid = github_organization.get_organization_sfid()
# Validate user is authorized for this SFDC ID.
can_access = check_user_authorization(auth_user, organization_sfid)
if not can_access['valid']:
return can_access['errors']
# Find all repositories that are under this organization
repositories = Repository().get_repositories_by_organization(organization_name)
for repository in repositories:
repository.delete()
github_organization.delete()
return {'success': True}
def get_organization_repositories(organization_name):
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
if github_organization.get_organization_installation_id() is not None:
cla.log.debug('GitHub Organization ID: {}'.format(
github_organization.get_organization_installation_id()))
installation = GitHubInstallation(
github_organization.get_organization_installation_id())
if installation.repos:
repos = []
for repo in installation.repos:
repos.append(repo.full_name)
return repos
else:
cla.log.debug(
'No repositories found for Github installation id: {}'.
format(github_organization.
get_organization_installation_id()))
return []
except DoesNotExist as err:
cla.log.warning(
'organization name {} does not exist, error: {}'.format(
organization_name, err))
return {'errors': {'organization_name': str(err)}}
def get_organization_by_sfid(auth_user: AuthUser, sfid):
# Check if user has permissions
user_permissions = UserPermissions()
try:
user_permissions.load(auth_user.username)
except DoesNotExist as err:
cla.log.warning('user {} does not exist, error: {}'.format(
auth_user.username, err))
return {'errors': {'user does not exist': str(err)}}
user_permissions_json = user_permissions.to_dict()
authorized_projects = user_permissions_json.get('projects')
if sfid not in authorized_projects:
cla.log.warning(
'user {} is not authorized for this Salesforce ID: {}'.format(
auth_user.username, sfid))
return {
'errors': {
'user is not authorized for this Salesforce ID.': str(sfid)
}
}
# Get all organizations under an SFDC ID
try:
organizations = get_github_organization_instance(
).get_organization_by_sfid(sfid)
except DoesNotExist as err:
cla.log.warning('sfid {} does not exist, error: {}'.format(sfid, err))
return {'errors': {'sfid': str(err)}}
return [organization.to_dict() for organization in organizations]
def get_organizations():
"""
Returns a list of github organizations in the CLA system.
:return: List of github organizations in dict format.
:rtype: [dict]
"""
return [github_organization.to_dict() for github_organization in get_github_organization_instance().all()]
def get_organization(organization_name):
"""
Returns the CLA github organization requested by Name.
:param organization_name: The github organization Name.
:type organization_name: Name
:return: dict representation of the github organization object.
:rtype: dict
"""
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
except DoesNotExist as err:
return {'errors': {'organization_name': str(err)}}
return github_organization.to_dict()
def get_organization_model(organization_name) -> Optional[GitHubOrg]:
"""
Returns a GitHubOrg model based on the the CLA github organization name.
:param organization_name: The github organization name.
:type organization_name: str
:return: model representation of the github organization object.
:rtype: GitHubOrg
"""
github_organization = get_github_organization_instance()
try:
cla.log.debug(f'Loading GitHub by organization name: {organization_name}..')
org = github_organization.get_organization_by_lower_name(organization_name)
cla.log.debug(f'Loaded GitHub by organization name: {org}')
return org
except DoesNotExist as err:
cla.log.warning(f'organization name {organization_name} does not exist, error: {err}')
return None
def get_organization(organization_name):
"""
Returns the CLA github organization requested by Name.
:param organization_name: The github organization Name.
:type organization_name: Name
:return: dict representation of the github organization object.
:rtype: dict
"""
github_organization = get_github_organization_instance()
try:
cla.log.debug(f'Loading GitHub by organization name: {organization_name}..')
org = github_organization.get_organization_by_lower_name(organization_name)
cla.log.debug(f'Loaded GitHub by organization name: {org}')
except DoesNotExist as err:
cla.log.warning(f'organization name {organization_name} does not exist')
return {'errors': {'organization_name': str(err)}}
return org.to_dict()
def get_organization_repositories(organization_name):
github_organization = get_github_organization_instance()
try:
github_organization.load(str(organization_name))
if github_organization.get_organization_installation_id() is not None:
print('GitHub Organization ID: {}'.format(
github_organization.get_organization_installation_id()))
installation = GitHubInstallation(
github_organization.get_organization_installation_id())
if installation.repos:
repos = []
for repo in installation.repos:
repos.append(repo.full_name)
return repos
else:
return []
except DoesNotExist as err:
return {'errors': {'organization_name': str(err)}}
corporate_document = get_document_instance()
corporate_document.set_document_name('Test CCLA Document')
corporate_document.set_document_file_id(str(uuid.uuid4()))
corporate_document.set_document_content_type('storage+pdf')
corporate_document.set_document_content(pdf_content, b64_encoded=False)
corporate_document.set_document_major_version(1)
corporate_document.set_document_minor_version(0)
corporate_document.set_raw_document_tabs(corporate_template.get_tabs())
project.add_project_corporate_document(corporate_document)
project.save()
## Create Github Org
GITHUB_ORGANIZATION_NAME = 'linuxfoundation'
GITHUB_INSTALLATION_ID = 72228 # NOT THE APP ID - find it in the webhook request JSON or URL when viewing installed apps.
cla.log.info('Creating GitHub Organization: %s' % GITHUB_ORGANIZATION_NAME)
github_org = get_github_organization_instance()
github_org.set_organization_name(GITHUB_ORGANIZATION_NAME)
github_org.set_organization_project_id(project.get_project_id())
# This will be different everytime the CLA app is installed.
github_org.set_organization_installation_id(GITHUB_INSTALLATION_ID)
github_org.save()
## User (For Company Management)
cla.log.info('Creating company manager user')
manager = get_user_instance()
manager.set_user_id(str(uuid.uuid4()))
manager.set_user_name('First User')
manager.set_user_email('*****@*****.**')
manager.set_user_email('*****@*****.**')
manager.set_user_github_id(123)
manager.save()
import urllib.request
from cla.utils import get_document_instance, get_github_organization_instance, get_project_instance, get_pdf_service
from cla.resources.contract_templates import CNCFTemplate
template = CNCFTemplate(document_type='Corporate',
major_version=1,
minor_version=0)
individual_template = CNCFTemplate(document_type='Individual',
major_version=1,
minor_version=0)
content = template.get_html_contract("", "")
pdf_generator = get_pdf_service()
pdf_content = pdf_generator.generate(content)
# Organisation
github_org = get_github_organization_instance(
).get_organization_by_installation_id(GITHUB_INSTALLATION_ID)
# Project
github_project1 = get_project_instance().get_projects_by_external_id(
PROJECT_EXTERNAL_ID1)[0]
github_project2 = get_project_instance().get_projects_by_external_id(
PROJECT_EXTERNAL_ID2)[0]
# Document
# ICLA Project1
individual_document = get_document_instance()
individual_document.set_document_name('Test ICLA Document')
individual_document.set_document_file_id(str(uuid.uuid4()))
individual_document.set_document_content_type('storage+pdf')
individual_document.set_document_content(pdf_content, b64_encoded=False)
individual_document.set_document_major_version(1)
individual_document.set_document_minor_version(0)
individual_document.set_raw_document_tabs(template.get_tabs())
def main(
aws_region,
rds_database,
rds_host,
rds_username,
rds_password,
rds_port,
dry_run,
):
"""
This script runs data migration from dynamodb to postgresql
"""
if os.environ.get("STAGE") is None:
logger.warning(
"Please set the 'STAGE' environment varaible - typically one of: {dev, staging, prod}"
)
return
stage = os.environ.get("STAGE")
if dry_run:
exit_cmd = input(
"This is a dry run. "
"You are running the script for the '{}' environment. "
'Press <ENTER> to continue ("exit" to exit): '.format(stage))
else:
exit_cmd = input(
"This is NOT a dry run. "
"You are running the script for the '{}' environment. "
'Press <ENTER> to continue ("exit" to exit): '.format(stage))
if exit_cmd == "exit":
return
start_time = datetime.now()
lock = threading.Lock()
threads = []
count = 0
for _ in range(IMPORT_THREADS):
connection = psycopg2.connect(
database=rds_database,
host=rds_host,
user=rds_username,
password=rds_password,
port=rds_port,
)
cursor = connection.cursor()
worker = Thread(target=import_to_pg,
args=(connection, cursor, tables_queues, dry_run))
worker.start()
threads.append(worker)
# TODO: cla models for signature,company-invitations and user-permissions
try:
user = utils.get_user_instance()
company = utils.get_company_instance()
project = utils.get_project_instance()
repository = utils.get_repository_instance()
github_orgs = utils.get_github_organization_instance()
gerrit = utils.get_gerrit_instance()
#signature = utils.get_signature_instance()
update_queue(user.all())
update_queue(company.all())
update_queue(project.all())
#update_queue(signature.all())
update_queue(repository.all())
update_queue(gerrit.all())
update_queue(github_orgs.all())
# block until all tasks are done
tables_queues.join()
except Exception as err:
logger.error(err)
# End workers
for _ in range(IMPORT_THREADS):
tables_queues.put(None)
for thread in threads:
thread.join()
duration = datetime.now() - start_time
logger.info(
"Data migration to the pg database run for a duration of {} ".format(
duration))
