def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
data = loads(request.body.decode())
branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
gh_repo = project.github()
ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
logger.debug(f'{namespace.slug}/{project.slug}: Pipeline #{pipeline_id} on commit {commit} for branch {branch}, '
f'status: {gl_status}')
# Report the status to Github
if gl_status in ['pending', 'success', 'failed']:
gh_status = gl_status if gl_status != 'failed' else 'failure'
if branch.startswith('pr/'):
gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
else:
try:
gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
except github.GithubException as e:
if e.status == 404:
# Happens when a new branch is created on gitlab and the pipeline event comes before the push event
logger.warning(f"Branch {branch} does not exist on github, unable to report the pipeline status.")
else:
raise
return HttpResponse(rep)
def get_thumbnail_path(instance, filename):
"""Custom name for uploaded thumbnails."""
group_name = slugify(str(instance.group.name))
project_name = slugify(str(instance.title))
ext = filename.split('.')[-1]
filename = '{}_{}.{}'.format(group_name, project_name, ext)
return os.path.join('thumbnails', filename)
def get_project_file_path(instance, filename):
"""Custom name for uploaded project files."""
group_name = slugify(str(instance.group.name))
project_name = slugify(str(instance.title))
filename = '{}_{}.qgs'.format(group_name, project_name)
return os.path.join('projects', filename)
def setUpClass(cls):
"""Run only once before the tests to avoid getting the objects in each test."""
super(GhTests, cls).setUpClass()
cls.namespace, _ = Namespace.objects.get_or_create(
name='gsaurel',
slug_gitlab=slugify('gsaurel'),
slug_github=slugify('hrp2-14'))
cls.project, _ = Project.objects.get_or_create(
name='rainboard-tests',
main_namespace=Namespace.objects.get(name='gsaurel'),
main_forge=Forge.objects.get(name='Gitlab'))
cls.github = cls.project.github()
cls.gitlab = cls.project.gitlab()
# Cleanup previous test
for pr in cls.github.get_pulls(state="open"):
pr.edit(state="closed")
gh_branches = [b.name for b in cls.github.get_branches()]
gl_branches = [b.name for b in cls.gitlab.branches.list()]
for branch_name in gl_branches:
if branch_name.startswith('pr/'):
cls.gitlab.branches.delete(branch_name)
for branch_name in ['test-branch-gitlab', 'test-branch-github']:
if branch_name in gh_branches:
cls.github.get_git_ref(f'heads/{branch_name}').delete()
if branch_name in gl_branches:
cls.gitlab.branches.delete(branch_name)
def push(request: HttpRequest, rep: str) -> HttpResponse:
"""Someone pushed on github. Synchronise local repo & gitlab."""
data = loads(request.body.decode())
namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
ref_s = data['ref'][11:] # strip 'refs/heads/'
print(f'push detected on github: {ref_s}')
gh_remote_s = f'github/{namespace.slug}'
gl_remote_s = f'gitlab/{namespace.slug}'
gh_ref_s = f'{gh_remote_s}/{ref_s}'
gl_ref_s = f'{gl_remote_s}/{ref_s}'
git_repo = project.git()
gh_remote = git_repo.remotes[gh_remote_s]
gh_remote.fetch()
gh_ref = gh_remote.refs[ref_s]
if data['after'] == "0000000000000000000000000000000000000000":
print("branch deleted")
for ref in [gh_ref_s, gl_ref_s, ref_s]:
if ref in git_repo.branches:
git_repo.delete_head(ref, force=True)
gitlab = Forge.objects.get(slug='gitlab')
project_u = f'{namespace.slug}/{project.slug}'.replace('/', '%2F')
branch_u = ref_s.replace('/', '%2F')
url = f'/projects/{project_u}/repository/branches/{branch_u}'
requests.delete(gitlab.api_url() + url, verify=gitlab.verify, headers=gitlab.headers())
return HttpResponse(rep)
if str(gh_ref.commit) != data['after']:
fail = f'push: wrong commit: {gh_ref.commit} vs {data["after"]}'
print(fail)
return HttpResponseBadRequest(fail)
if gh_ref_s in git_repo.branches:
git_repo.branches[gh_ref_s].commit = data['after']
else:
git_repo.create_head(gh_ref_s, commit=data['after'])
if ref_s in git_repo.branches:
git_repo.branches[ref_s].commit = data['after']
else:
git_repo.create_head(ref_s, commit=data['after'])
if gl_remote_s not in git_repo.remotes:
print(f'project {project} not available on {gl_remote_s}')
return HttpResponse(rep)
if gl_ref_s in git_repo.branches:
git_repo.branches[gl_ref_s].commit = data['after']
else:
git_repo.create_head(gl_ref_s, commit=data['after'])
gl_remote = git_repo.remotes[gl_remote_s]
gl_remote.fetch()
if ref_s not in gl_remote.refs or str(gl_remote.refs[ref_s].commit) != data['after']:
print(f'pushing {data["after"]} on {ref_s} on gitlab')
gl_remote.push(ref_s)
return HttpResponse(rep)
def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
print('pipeline')
data = loads(request.body.decode())
namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
branch, commit, status = (data['object_attributes'][key] for key in ['ref', 'sha', 'status'])
# status in ['pending', 'running', 'success', 'failed']
print(namespace, project, branch, commit, status)
return HttpResponse(rep)
def save(self, force_insert=False, force_update=False, using=None,
update_fields=None):
try:
if self.category == "training":
message = MailerMessage()
context = { 'user': self.applicant, }
body = render_to_string("events/training_tracker.txt", context)
message.subject = " ".join(self.organizers.all())+"organized a training: " + self.target.name +"EOM"
message.content = "end of message"
message.to_address = "*****@*****.**"
message.save()
except:
pass
self.slug = slugify(self.name)
if (self.feedbacksheet):
if self.pk:
# if created
orig = Event.objects.get(pk=self.pk)
if orig.feedbacksheet != self.feedbacksheet:
#if feedbacksheet has changed
for pax in self.participation_set.all():
if pax.feedback:
pax.feedback.delete()
pax.feedback = create_answer_set(self.feedbacksheet)
pax.save()
if (self.questionaire):
if self.pk:
orig = Event.objects.get(pk=self.pk)
if orig.questionaire != self.questionaire:
for application in self.applicants.all():
application.questionaire = create_answer_set(self.questionaire)
application.save()
super(Event, self).save(force_insert, force_update, using, update_fields)
class OrganizationFactory(factory.django.DjangoModelFactory):
name = factory.Sequence(lambda n: f"org-{n}")
slug = factory.LazyAttribute(lambda obj: slugify(obj.name))
customer = factory.RelatedFactory(
"squarelet.organizations.tests.factories.CustomerFactory",
"organization")
class Meta:
model = "organizations.Organization"
django_get_or_create = ("name", )
@factory.post_generation
def users(self, create, extracted, **kwargs):
# pylint: disable=unused-argument
if create and extracted:
for user in extracted:
MembershipFactory(user=user, organization=self, admin=False)
@factory.post_generation
def admins(self, create, extracted, **kwargs):
# pylint: disable=unused-argument
if create and extracted:
for user in extracted:
MembershipFactory(user=user, organization=self, admin=True)
@factory.post_generation
def plans(self, create, extracted, **kwargs):
# pylint: disable=unused-argument
if create and extracted:
for plan in extracted:
SubscriptionFactory(plan=plan, organization=self)
def save(self, force_insert=False, force_update=False, using=None,
update_fields=None):
self.slug = slugify(self.name)
if (self.feedbacksheet):
if self.pk:
# if created
orig = Event.objects.get(pk=self.pk)
if orig.feedbacksheet != self.feedbacksheet:
#if feedbacksheet has changed
for pax in self.participation_set.all():
if pax.feedback:
pax.feedback.delete()
pax.feedback = create_answer_set(self.feedbacksheet)
pax.save()
if (self.questionaire):
if self.pk:
orig = Event.objects.get(pk=self.pk)
if orig.questionaire != self.questionaire:
for application in self.applicants.all():
application.questionaire = create_answer_set(self.questionaire)
application.save()
super(Event, self).save(force_insert, force_update, using, update_fields)
privileged, created = Group.objects.get_or_create(name=self.slug + "_privileged")
privileged.save()
privileged.user_set = self.organizers.all()
for oc in self.organizing_committee.all():
assign_perm('change_event', Group.objects.get(name=oc.slug + "_privileged"),
self)
privileged.save()
assign_perm('change_event', privileged, self)
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
self.slug = slugify(self.name)
if (self.feedbacksheet):
if self.pk:
# if created
orig = Event.objects.get(pk=self.pk)
if orig.feedbacksheet != self.feedbacksheet:
#if feedbacksheet has changed
for pax in self.participation_set.all():
if pax.feedback:
pax.feedback.delete()
pax.feedback = create_answer_set(self.feedbacksheet)
pax.save()
if (self.questionaire):
if self.pk:
orig = Event.objects.get(pk=self.pk)
if orig.questionaire != self.questionaire:
for application in self.applicants.all():
application.questionaire = create_answer_set(
self.questionaire)
application.save()
super(Event, self).save(force_insert, force_update, using,
update_fields)
def save(self, *args, **kwargs):
try:
self.slug = slugify(self.name)
geocoder = Geocoder()
address = self.address
results, status_code = geocoder.geocode({'address': self.name})
self.lat, self.lng = results[0]['geometry']['location']['arg']
except:
pass
super(Team, self).save(*args, **kwargs)
privileged, created = Group.objects.get_or_create(name=self.slug + "_privileged")
privileged.save()
privileged.user_set = self.privileged()
privileged.save()
assign_perm('change_team', privileged, self)
if not self.pk:
a = Event.objects.create(
name=str(self.slug + " recruitment"),
scope="local",
category="recruitment",
description="We are always recruiting and welcoming new people.",
start_date=datetime.now()
)
a.save()
a.organizing_committee = [self]
def save(self, *args, **kwargs):
try:
self.slug = slugify(self.name)
geocoder = Geocoder()
address = self.address
results, status_code = geocoder.geocode({'address': self.name})
self.lat, self.lng = results[0]['geometry']['location']['arg']
except:
pass
super(Team, self).save(*args, **kwargs)
privileged, created = Group.objects.get_or_create(name=self.slug +
"_privileged")
privileged.save()
privileged.user_set = self.privileged()
privileged.save()
assign_perm('change_team', privileged, self)
if not self.pk:
a = Event.objects.create(
name=str(self.slug + " recruitment"),
scope="local",
category="recruitment",
description=
"We are always recruiting and welcoming new people.",
start_date=datetime.now())
a.save()
a.organizing_committee = [self]
def generate_slug(cls, title, instance_pk=None):
original_slug = slugify(title)
return generate_unique_slug(
cls._meta.get_field_by_name('slug')[0],
cls.all_objects.filter(pk=instance_pk).first() or cls(),
original_slug,
cls.all_objects
)
def upload_path(instance, filename):
"""
For setting the upload_to parameter of FileFields and ImageFields,
e.g. field_name = FileField(upload_to=upload_path)
Returns a MEDIA_ROOT file path that is quite descriptive as it uses
the instance.__str__ method. It is assumed that this method is defined.
"""
return os.path.join(instance.__class__.__name__, slugify(str(instance)), filename)
class EntitlementFactory(factory.django.DjangoModelFactory):
name = factory.Sequence(lambda n: f"Entitlement {n}")
slug = factory.LazyAttribute(lambda obj: slugify(obj.name))
client = factory.SubFactory("squarelet.oidc.tests.factories.ClientFactory")
description = factory.Sequence(lambda n: f"Description {n}")
class Meta:
model = "organizations.Entitlement"
def build_username(self, email):
username = slugify(email).lower()
if len(username) > 30:
username = username[:30]
lookup = {"username__iexact": username}
try:
User.objects.exclude(id=self.form.instance.id).get(**lookup)
except User.DoesNotExist:
return username
def build_named_menu_nodes(menu_name_or_slug,
page_nodes,
draft_mode_active,
language,
namespace=None):
# Return if no nodes!
if not page_nodes:
return
# Get the name and derive the slug - for the cache key
menu_slug = slugify(menu_name_or_slug)
logger.debug(u'Creating Named Menu: "{}"'.format(menu_slug))
# Get named menu from cache if available, no caching in edit mode!~
# --------------------------------
named_menu = None
if not draft_mode_active:
named_menu = cache.get(menu_slug, language)
if named_menu:
return named_menu
# Rebuild named menu if not cached and not in draft mode - post-cut/levels/etc happens after
# --------------------------------
# Get by Slug or from Menu name - backwards compatible
try:
named_menu = CMSNamedMenu.objects.get(slug__exact=menu_slug).pages
except ObjectDoesNotExist:
try:
named_menu = CMSNamedMenu.objects.get(
name__iexact=menu_name_or_slug).pages
except ObjectDoesNotExist:
logger.info(u'Named menu with name(slug): "%s" not found',
menu_name_or_slug)
# If we get the named menu, build the nodes
if named_menu:
named_menu_nodes = []
# Convert to draft mode if required
if draft_mode_active:
convert_menu_to_draft_mode(named_menu)
for item in named_menu:
# Loops through each [{'id':[page_id], 'children':[,,]},... etc
# Maps nodes and child nodes
node = create_node(item, page_nodes, level=0, namespace=namespace)
if node is not None:
named_menu_nodes.append(node)
# Cache named menu to avoid repeated queries
if not draft_mode_active:
cache.set(menu_slug, named_menu_nodes, language)
return named_menu_nodes
def course_file_path(instance: 'CourseUpload', filename: str) -> str:
"""
Returns the file path of a user uploaded course file, used in the upload_to
parameter of the FileField.
"""
return os.path.join(
'fagfiler',
slugify(instance.course.course_code).upper(),
filename,
)
def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's check suites."""
data = loads(request.body.decode())
slug = slugify(data['repository']['name'])
if 'ros-release' in slug: # Don't run check suites on ros-release repositories
return HttpResponse(rep)
models.GithubCheckSuite.objects.get_or_create(id=data['check_suite']['id'])
return HttpResponse(rep)
class PlanFactory(factory.django.DjangoModelFactory):
"""A factory for creating Plan test objects"""
name = factory.Sequence(lambda n: f"Plan {n}")
slug = factory.LazyAttribute(lambda obj: slugify(obj.name))
public = True
class Meta:
model = "organizations.Plan"
django_get_or_create = ("name", )
def test_room_creation(self):
assert self.client.login(username="******", password="******")
data = {
'name': "new room",
'password': "",
'moderator': ""
}
room_slug = slugify(data['name'])
response = self.client.post(reverse("room_new"), data)
self.assertRedirects(response, reverse("room_view", kwargs={'room_slug': room_slug}))
self.assertEqual(Room.objects.filter(name="new room", pinned=False).count(), 1)
def get_upload_path(instance, filename):
ext = filename.split('.')[-1]
img_name = "{}.{}".format(slugify(instance.title), ext)
fullname = os.path.join(settings.MEDIA_ROOT, 'notes', img_name)
#Remove any previous files with same name.
if os.path.exists(fullname):
os.remove(fullname)
return img_name
def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's Pull Requests."""
data = loads(request.body.decode())
namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
git_repo = project.git()
remote_s = f'github/{data["pull_request"]["head"]["repo"]["owner"]["login"]}'
if remote_s not in git_repo.remotes:
git_repo.create_remote(remote_s, data["pull_request"]["head"]["repo"]["clone_url"])
remote = git_repo.remotes[remote_s]
remote.fetch()
branch = f'pr/{data["number"]}'
commit = data['pull_request']['head']['sha']
if branch in git_repo.branches:
git_repo.heads[branch] = commit
else:
git_repo.create_head(branch, commit=commit)
print(f'pushing {commit} on {branch} on gitlab')
gl_remote = git_repo.remotes[f'gitlab/{namespace.slug}']
gl_remote.push(branch)
return HttpResponse(rep)
def clean_shop_name(self):
shop_name = self.cleaned_data['shop_name']
shop_exists = True
try:
Shop.objects.get(name__iexact=shop_name)
except ObjectDoesNotExist:
shop_exists = False # return shop_name # if shop_name doesn't exist, this is good. We can create the shop
if shop_exists:
raise forms.ValidationError('Shop name is already taken.')
# if the shop name resolves to a view that is not the shop.. it is trying to use a url that we already use.. dont let this happen!
if len(shop_name) > 0 and resolve(urlparse.urlparse('/' + slugify(shop_name.lower()) + '/')[2]).view_name != 'designer_shop.views.shopper':
raise forms.ValidationError('Not a valid shop name, please choose another')
return shop_name
def title_to_slug(title):
"""
Get a significant word or two from a puzzle title, for use as a
clean identifier, such as for a channel name.
"""
slug1 = slugify(title)
parts = slug1.split('-')
new_parts = []
for part in parts:
if part not in COMMON_WORDS:
new_parts.append(part)
if len(part) >= 5 or len(new_parts) >= 2:
break
if not new_parts:
new_parts = parts[:2]
return '-'.join(new_parts)
def handle(self, *args, **options):
if not settings.LOCAL_GROUPS_AIRTABLE:
raise base.CommandError("Airtable environment variables are unset")
local_group_types = {
"City": models.LocalGroupType.CITY,
"Country": models.LocalGroupType.COUNTRY,
"University": models.LocalGroupType.UNIVERSITY,
}
local_groups = []
for table_name, is_active in [("Active", True), ("Inactive", False)]:
for record in airtable.Airtable(
table_name=table_name,
**settings.LOCAL_GROUPS_AIRTABLE).get_all():
airtable_fields = record["fields"]
name = airtable_fields["Group Name"]
city_or_town = airtable_fields.get("City or Town", "")
country = airtable_fields.get("Country", "")
local_group = models.LocalGroup(
# This will raise an IntegrityError on collisions
slug=autoslug_utils.slugify(name),
name=name,
is_active=is_active,
local_group_type=local_group_types.get(
airtable_fields.get("University or City")),
city_or_town=city_or_town,
country=country,
website=airtable_fields.get("Website", ""),
facebook_group=airtable_fields.get("Facebook Group", ""),
facebook_page=airtable_fields.get("Facebook Page", ""),
email=airtable_fields.get(
"Official Group Email",
airtable_fields.get(
"LEAN email", airtable_fields.get("EAF email",
"")),
),
meetup_url=airtable_fields.get("Meetup.com", ""),
airtable_record=record["id"],
)
if city_or_town and country:
self.stdout.write(f"Geocoding: {city_or_town}, {country}")
time.sleep(1)
local_group.geocode()
local_groups.append(local_group)
models.LocalGroup.objects.bulk_create(local_groups)
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
try:
if self.category == "training":
message = MailerMessage()
context = {
'user': self.applicant,
}
body = render_to_string("events/training_tracker.txt", context)
message.subject = " ".join(self.organizers.all(
)) + "organized a training: " + self.target.name + "EOM"
message.content = "end of message"
message.to_address = "*****@*****.**"
message.save()
except:
pass
self.slug = slugify(self.name)
if (self.feedbacksheet):
if self.pk:
# if created
orig = Event.objects.get(pk=self.pk)
if orig.feedbacksheet != self.feedbacksheet:
#if feedbacksheet has changed
for pax in self.participation_set.all():
if pax.feedback:
pax.feedback.delete()
pax.feedback = create_answer_set(self.feedbacksheet)
pax.save()
if (self.questionaire):
if self.pk:
orig = Event.objects.get(pk=self.pk)
if orig.questionaire != self.questionaire:
for application in self.applicants.all():
application.questionaire = create_answer_set(
self.questionaire)
application.save()
super(Event, self).save(force_insert, force_update, using,
update_fields)
def save(self, *args, **kwargs):
try:
self.slug = slugify(self.name)
geocoder = Geocoder()
address = self.address
results, status_code = geocoder.geocode({'address': self.name})
self.lat, self.lng = results[0]['geometry']['location']['arg']
except:
pass
if not self.pk:
super(Team, self).save(*args, **kwargs)
a = Event.objects.create(
name=str(self.slug + " recruitment"),
scope="local",
category="recruitment",
description="We are always recruiting and welcoming new people.",
start_date=datetime.now()
)
a.save()
a.organizing_committee = [self]
else:
super(Team, self).save(*args, **kwargs)
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
self.slug = slugify(self.name)
if (self.feedbacksheet):
if self.pk:
# if created
orig = Event.objects.get(pk=self.pk)
if orig.feedbacksheet != self.feedbacksheet:
#if feedbacksheet has changed
for pax in self.participation_set.all():
if pax.feedback:
pax.feedback.delete()
pax.feedback = create_answer_set(self.feedbacksheet)
pax.save()
if (self.questionaire):
if self.pk:
orig = Event.objects.get(pk=self.pk)
if orig.questionaire != self.questionaire:
for application in self.applicants.all():
application.questionaire = create_answer_set(
self.questionaire)
application.save()
super(Event, self).save(force_insert, force_update, using,
update_fields)
privileged, created = Group.objects.get_or_create(name=self.slug +
"_privileged")
privileged.save()
privileged.user_set = self.organizers.all()
for oc in self.organizing_committee.all():
assign_perm('change_event',
Group.objects.get(name=oc.slug + "_privileged"), self)
privileged.save()
assign_perm('change_event', privileged, self)
def slug(self):
return slugify(self.display_name)
def save(self, *args, **kwargs):
self.slug = slugify(self.name)
super(Category, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
self.slug = slugify(self.name)
super(Business, self).save(*args, **kwargs)
def gen_slug(apps, schema_editor):
max_length = Product._meta.get_field('slug').max_length
for row in Product.objects.all():
row.slug = slugify(row.name)[:max_length]
row.save()
def generate_slug(instance):
name = instance.name
return slugify(name)
def when_i_click_on_the_item(step, itemname):
theitem = wait_for_element_with_css_selector_to_be_displayed(
"a[href='/demo/" + slugify(itemname) + "']")
step.scenario.context['itemurl'] = theitem.get_attribute("href")
theitem.click()
def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's Pull Requests."""
data = loads(request.body.decode())
event = data['action']
branch = f'pr/{data["number"]}'
login = slugify(data["pull_request"]["head"]["repo"]["owner"]["login"])
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
git_repo = project.git()
logger.debug(f'{namespace.slug}/{project.slug}: Pull request on {branch}: {event}')
# Prevent pull requests on master when necessary
if event in ['opened', 'reopened']:
gh = project.github()
pr = gh.get_pull(data["number"])
pr_branch = pr.base.ref
if not project.accept_pr_to_master and pr_branch == 'master' \
and 'devel' in [b.name for b in gh.get_branches()] and login != namespace.slug_github:
logger.info(f"{namespace.slug}/{project.slug}: New pr {data['number']} to master")
pr.create_issue_comment("Hi ! This project doesn't usually accept pull requests on master. If this wasn't "
"intentionnal, you can change the base branch of this pull request to devel "
"(No need to close it for that). Best, a bot.")
gh_remote_name = f'github/{login}'
if gh_remote_name not in git_repo.remotes:
remote = git_repo.create_remote(gh_remote_name, data["pull_request"]["head"]["repo"]["clone_url"])
else:
remote = git_repo.remote(gh_remote_name)
# Sync the pull request with the pr/XX branch on Gitlab
if event in ['opened', 'reopened', 'synchronize']:
remote.fetch()
commit = data['pull_request']['head']['sha']
# Update branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
# Push the changes to gitlab
logger.info(f'{namespace.slug}/{project.slug}: Pushing {commit} on {branch} on gitlab')
try:
git_repo.git.push(gl_remote_name, branch)
except git.exc.GitCommandError:
logger.warning(f'{namespace.slug}/{project.slug}: Failed to push on {branch} on gitlab, force pushing ...')
git_repo.git.push(gl_remote_name, branch, force=True)
# The pull request was closed, delete the branch pr/XX on Gitlab
elif event == 'closed':
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
git_repo.delete_remote(gh_remote_name)
project.gitlab().branches.delete(branch)
logger.info(f'{namespace.slug}/{project.slug}: Deleted branch {branch}')
return HttpResponse(rep)
def get_context(self, context, **kwargs):
# Get the name and derive the slug - for the cache key
menu_name = kwargs.pop('menu_name')
menu_slug = slugify(menu_name)
context.update({
'children': [],
'template': kwargs.get('template'),
'from_level': kwargs.get('from_level'),
'to_level': kwargs.get('to_level'),
'extra_inactive': kwargs.get('extra_inactive'),
'extra_active': kwargs.get('extra_active'),
'namespace': kwargs.get('namespace')
})
lang = get_language()
request = context['request']
namespace = kwargs['namespace']
root_id = kwargs['root_id']
# Try to get from Cache first
arranged_nodes = cache.get(menu_slug, lang)
# Create menu from Json if not
if arranged_nodes is None:
logger.debug(u'Creating menu "%s %s"', menu_slug, lang)
named_menu = None
arranged_nodes = []
# Get by Slug or from Menu name - backwards compatible
try:
named_menu = CMSNamedMenu.objects.get(
slug__exact=menu_slug).pages
except ObjectDoesNotExist:
try:
named_menu = CMSNamedMenu.objects.get(
name__iexact=menu_name).pages
except ObjectDoesNotExist:
logger.info(
u'Named menu with name(slug): "%s (%s)" not found',
menu_name, lang)
# If we get the named menu, build the nodes
if named_menu:
# Try to get all the navigation nodes from the cache, or repopulate if not
nodes = getattr(request, NODES_REQUEST_CACHE_ATTR, None)
if nodes is None:
nodes = get_nodes(request, namespace, root_id)
# getting nodes is slow, cache on request object will
# speedup if more than one named menus are on the page
setattr(request, NODES_REQUEST_CACHE_ATTR, nodes)
# Get the named menu nodes
arranged_nodes = self.arrange_nodes(nodes,
named_menu,
namespace=namespace)
if len(arranged_nodes) > 0:
logger.debug(u'Put %i menu "%s %s" to cache',
len(arranged_nodes), menu_slug, lang)
cache.set(menu_slug, lang, arranged_nodes)
else:
logger.debug(u'Don\'t cache empty "%s %s" menu!',
menu_slug, lang)
else:
logger.debug(u'Fetched menu "%s %s" from cache', menu_slug, lang)
context.update({'children': arranged_nodes})
return context
def add_slug(apps, schema_editor):
Project = apps.get_model('projects', 'Project')
for project in Project.objects.all():
if not project.slug:
project.slug = slugify(project.title)
project.save()
def generated_slug(self):
return slugify(self.title)
def build_username(self, email):
username_from_email = slugify(email).lower()
if len(username_from_email) > 30:
username_from_email = username_from_email[:30]
return username_from_email
def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
"""Someone pushed on github or gitlab. Synchronise local & remote repos."""
data = loads(request.body.decode())
slug = slugify(data['repository']['name'])
if 'ros-release' in slug: # Don't sync ros-release repositories
return HttpResponse(rep)
if source == SOURCES.gitlab:
namespace = get_object_or_404(Namespace,
slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
else:
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slug)
branch = data['ref'][11:] # strip 'refs/heads/'
commit = data['after']
gl_remote_name = f'gitlab/{namespace.slug}'
gh_remote_name = f'github/{namespace.slug}'
git_repo = project.git()
logger.debug(f'{namespace.slug}/{slug}: Push detected on {source.name} {branch} (commit {commit})')
if branch.startswith('pr/'): # Don't sync pr/XX branches here, they are already handled by pull_request()
return HttpResponse(rep)
if branch.startswith('release/'): # Don't sync release/X.Y.Z branches at all
return HttpResponse(rep)
# Fetch the latest commit from gitlab
if gl_remote_name in git_repo.remotes:
gl_remote = git_repo.remote(gl_remote_name)
else:
gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
gl_remote.fetch()
# Fetch the latest commit from github
if gh_remote_name in git_repo.remotes:
gh_remote = git_repo.remote(gh_remote_name)
else:
gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
gh_remote.fetch()
# The branch was deleted on one remote, delete the branch on the other remote as well
if commit == "0000000000000000000000000000000000000000":
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
if source == SOURCES.gitlab:
project.github().get_git_ref(f'heads/{branch}').delete()
else:
project.gitlab().branches.delete(branch)
logger.info(f'{namespace.slug}/{slug}: Deleted branch {branch}')
return HttpResponse(rep)
# Make sure we fetched the latest commit
ref = gl_remote.refs[branch] if source == SOURCES.gitlab else gh_remote.refs[branch]
if str(ref.commit) != commit:
fail = f'Push: wrong commit: {ref.commit} vs {commit}'
logger.error(f'{namespace.slug}/{slug}: ' + fail)
return HttpResponseBadRequest(fail)
# Update the branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Push the changes to other remote
try:
if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on github')
git_repo.git.push(gh_remote_name, branch)
elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on gitlab')
git_repo.git.push(gl_remote_name, branch)
else:
return HttpResponse('already synced')
except git.exc.GitCommandError:
# Probably failed because of a force push
logger.exception(f'{namespace.slug}/{slug}: Forge sync failed')
message = traceback.format_exc()
message = re.sub(r'://.*@', '://[REDACTED]@', message) # Hide access tokens in the mail
mail_admins(f'Forge sync failed for {namespace.slug}/{slug}', message)
return HttpResponse(rep)
def get_context(self, context, **kwargs):
# Get the name and derive the slug - for the cache key
menu_name = kwargs.pop('menu_name')
menu_slug = slugify(menu_name)
context.update({
'children': [],
'template': kwargs.get('template'),
'from_level': kwargs.get('from_level'),
'to_level': kwargs.get('to_level'),
'extra_inactive': kwargs.get('extra_inactive'),
'extra_active': kwargs.get('extra_active'),
'namespace': kwargs.get('namespace')
})
lang = get_language()
request = context['request']
namespace = kwargs['namespace']
root_id = kwargs['root_id']
# Try to get from Cache first
arranged_nodes = cache.get(menu_slug, lang)
# Create menu from Json if not
if arranged_nodes is None:
logger.debug(u'Creating menu "%s %s"', menu_slug, lang)
named_menu = None
arranged_nodes = []
# Get by Slug or from Menu name - backwards compatible
try:
named_menu = CMSNamedMenu.objects.get(slug__exact=menu_slug).pages
except ObjectDoesNotExist:
try:
named_menu = CMSNamedMenu.objects.get(name__iexact=menu_name).pages
except ObjectDoesNotExist:
logger.info(u'Named menu with name(slug): "%s (%s)" not found', menu_name, lang)
# If we get the named menu, build the nodes
if named_menu:
# Try to get all the navigation nodes from the cache, or repopulate if not
nodes = getattr(request, NODES_REQUEST_CACHE_ATTR, None)
if nodes is None:
nodes = get_nodes(request, namespace, root_id)
# getting nodes is slow, cache on request object will
# speedup if more than one named menus are on the page
setattr(request, NODES_REQUEST_CACHE_ATTR, nodes)
# Get the named menu nodes
arranged_nodes = self.arrange_nodes(nodes, named_menu, namespace=namespace)
if len(arranged_nodes) > 0:
logger.debug(u'Put %i menu "%s %s" to cache', len(arranged_nodes), menu_slug, lang)
cache.set(menu_slug, lang, arranged_nodes)
else:
logger.debug(u'Don\'t cache empty "%s %s" menu!', menu_slug, lang)
else:
logger.debug(u'Fetched menu "%s %s" from cache', menu_slug, lang)
context.update({
'children': arranged_nodes
})
return context
