def test_entity_description(self): link_to_school = Link( destination_content_type=ContentType.objects.get_for_model( self.school ), destination_object_id=self.school.id, ) self.assertEquals(link_to_school.__unicode__(), self.school.name) link_to_building = Link( destination_content_type = ContentType.objects.get_for_model(self.main_building), destination_object_id = self.main_building.id, ) self.assertEquals( link_to_building.__unicode__(), self.main_building.__unicode__() ) self.assertEquals( link_to_building.summary(), u"St Mary's Street, Cardiff" ) """
def import_from_old_site(request): user = request.user.get_profile() if request.method == 'POST': form = project_forms.ImportProjectForm(request.POST) if form.is_valid(): course = form.cleaned_data['course'] #CS - too much logic in view project = Project(name=course['name'], kind=course['kind'], short_description=course['short_description'], long_description=course['long_description'], imported_from=course['slug']) project.save() act = Activity(actor=user, verb=verbs['post'], scope_object=project, target_object=project) act.save() participation = Participation(project=project, user=user, organizing=True) participation.save() new_rel, created = Relationship.objects.get_or_create(source=user, target_project=project) new_rel.deleted = False new_rel.save() if course['detailed_description']: detailed_description_content = course['detailed_description'] else: detailed_description_content = render_to_string( "projects/detailed_description_initial_content.html", {'project': project}) detailed_description = Page(title=_('Full Description'), slug='full-description', content=detailed_description_content, listed=False, author_id=user.id, project_id=project.id) detailed_description.save() project.detailed_description_id = detailed_description.id sign_up = Signup(between_participants=course['sign_up'], author_id=user.id, project_id=project.id) sign_up.save() project.save() for title, content in course['tasks']: new_task = Page(title=title, content=content, author=user, project=project) new_task.save() for name, url in course['links']: new_link = Link(name=name, url=url, user=user, project=project) new_link.save() project.create() messages.success(request, _('The %s has been imported.') % project.kind.lower()) return http.HttpResponseRedirect(reverse('projects_show', kwargs={ 'slug': project.slug, })) else: msg = _("Problem importing the study group, course, ...") messages.error(request, msg) else: form = project_forms.ImportProjectForm() return render_to_response('projects/project_import.html', { 'form': form, 'import_tab': True}, context_instance=RequestContext(request))
def mutate(self, info, url, description): link = Link(url=url, description=description) link.save() return CreateLink(id=link.id, url=link.url, description=link.description)
def mutate_and_get_payload(root, info, **input): user = info.context.user or None link = Link(url=input.get('url'), description=input.get('description'), posted_by=user) link.save() return RelayCreateLink(link=link)
def mutate(self, info, url, description): user = info.context.user or None link = Link(url=url, description=description, posted_by=user) link.save() return CreateLink(id=link.id, url=link.url, description=link.description, posted_by=link.posted_by)
def mutate(self, info, url, description): user = get_user(info) or None link = Link(url=url, description=description, posted_by=user) ## save the date link.save() return CreateLink(id=link.id, url=link.url, description=link.description, posted_by=link.posted_by)
def mutate_and_get_payload(self, info, **input): user = info.context.user or None logger.info('12345 上山打老虎') link = Link( url=input.get('url'), description=input.get('description'), posted_by=user, ) link.save() return RelayCreateLink(link=link)
def mutate(self, info, url, description): # will fail if there is no logged in user posted_by = get_user(info) link = Link(url=url, description=description, posted_by=posted_by) link.save() return CreateLink( id=link.id, url=link.url, description=link.description, posted_by=link.posted_by, )
def test_link_encode(self): tests = [ (1, 'MQ=='), (100, 'MTAw'), (10000, 'MTAwMDA='), (1000000, 'MTAwMDAwMA=='), ] for test in tests: l = Link(id=test[0], url='http://example.com') key = l.encode() self.assertEqual(key, test[1])
def home(request): print("home called") if request.method == 'POST': if request.POST['urlinput']: _user = request.user.is_authenticated if _user: print("user-mode") print("request ", request.POST['urlinput']) link = Link() link.to = request.POST['urlinput'] result = False if _user: if request.POST['suffixinput']: link.suffix, result = createSuffix( request.POST['suffixinput']) else: result = True link.suffix = createRandomSuffix() else: result = True link.suffix = createRandomSuffix() link.is_active = True is_onetime_checkbox = request.POST.get('onetimeinput', False) if is_onetime_checkbox: link.redirects = 1 datecur = timezone.datetime.now() link.expiry = datetime(datecur.year + 100, datecur.month, datecur.day) if _user: link.owner = request.user user_account = Account.objects.get(username=request.user) user_account.linkset.append(link) user_account.save() else: link.owner = User.objects.get(username='******') link.save() retlink = "zuru.io/" + link.suffix error = "Sorry! That URL is not available." if not result: return render(request, 'links/home.html', { 'retlink1': retlink, 'error': error }) else: return render(request, 'links/home.html', {'retlink1': retlink}) else: return render(request, 'links/home.html')
def edit_url(request: HttpRequest, name: str, context: dict): link = None if name is not None: try: link = Link.objects.get(name=name) except Exception: raise Http404 context['link'] = link if request.POST: formname = request.POST['name'] url = request.POST['url'] try: request.POST['instant'] instant = True except: instant = False nullcheck = assure_formname_and_url_not_null(formname, url, request, context) if (nullcheck): return nullcheck del nullcheck formname = formname.replace(" ", "_") name_in_use = assure_formname_is_not_in_use(formname, name, request, context) if (name_in_use): return name_in_use del name_in_use try: URLValidator(None).__call__(url) except ValidationError: context = dangeralert(context, 'Thats not a valid URL!') return render(request, 'dashlink.html', context) try: if link is None: link = Link(url=url, name=formname, instant_redirect=instant) else: link.url = url link.name = formname link.instant_redirect = instant link.save() except Exception as e: print(e) context = dangeralert(context, 'An error has occured!') return render(request, 'dashlink.html', context) return HttpResponseRedirect(reverse('dashboard:index')) return HttpResponse("You can only POST this page.")
def token(token): token = Token.query.filter_by(owner=current_user, identity=token).limit(1).first() if not token: raise NotFound('Token not found.') form = AddLinkForm() if form.submit.data: if form.validate_on_submit(): link = Link(token, form.url.data) db.session.add(link) db.session.commit() # Clears form and allows page refresh without re-submitting the form. return redirect(url_for('token', token=token.identity)) for k, v in form.errors.items(): for m in v: flash(m, 'danger') return render_template('token.html', token=token, form=form)
def create_or_update_single_bill(self, proposal, pdf_link, link_file): if not(proposal['date']) or CUTOFF_DATE and proposal['date'] < CUTOFF_DATE: return law_name = proposal['law'] (law, created) = Law.objects.get_or_create(title=law_name) if created: law.save() if law.merged_into: law = law.merged_into title = u'' if proposal['correction']: title += proposal['correction'] if proposal['comment']: title += ' ' + proposal['comment'] if len(title)<=1: title = u'חוק חדש' (gp,created) = GovProposal.objects.get_or_create(booklet_number=proposal['booklet'], knesset_id=18, source_url=proposal['link'], title=title, law=law, date=proposal['date']) if created: gp.save() logger.debug("created GovProposal id = %d" % gp.id) bill_params = dict(law=law, title=title, stage='3', stage_date=proposal['date']) similar_bills = Bill.objects.filter(**bill_params).order_by('id') if len(similar_bills) >= 1: b = similar_bills[0] if len(similar_bills) > 1: logger.debug("multiple bills detected") for bill in similar_bills: if bill.id == b.id: logger.debug("bill being used now - %d" % bill.id) else: logger.debug("bill with same fields - %d" % bill.id) else: b = Bill(**bill_params) b.save() gp.bill = b gp.save() if link_file.link is None: link = Link(title=pdf_link, url=pdf_link, content_type=ContentType.objects.get_for_model(gp), object_pk=str(gp.id)) link.save() link_file.link = link link_file.save() logger.debug("check updated %s" % b.get_absolute_url())
def mutate(root, input, context, info): user = get_user(context) or None link = Link( url=input.get('url'), description=input.get('description'), posted_by=user, ) link.save() return CreateLink( id=link.id, url=link.url, description=link.description, posted_by=link.posted_by, )
def clone(request): user = request.user.get_profile() if request.method == 'POST': form = project_forms.CloneProjectForm(request.POST) if form.is_valid(): base_project = form.cleaned_data['project'] #CS - too much logic in view project = Project(name=base_project.name, category=base_project.category, other=base_project.other, other_description=base_project.other_description, short_description=base_project.short_description, long_description=base_project.long_description, clone_of=base_project) project.save() act = Activity(actor=user, verb=verbs['post'], scope_object=project, target_object=project) act.save() participation = Participation(project=project, user=user, organizing=True) participation.save() new_rel, created = Relationship.objects.get_or_create(source=user, target_project=project) new_rel.deleted = False new_rel.save() detailed_description = Page(title=_('Full Description'), slug='full-description', content=base_project.detailed_description.content, listed=False, author_id=user.id, project_id=project.id) detailed_description.save() project.detailed_description_id = detailed_description.id base_sign_up = base_project.sign_up.get() sign_up = Signup(public=base_sign_up.public, between_participants=base_sign_up.between_participants, author_id=user.id, project_id=project.id) sign_up.save() project.save() tasks = Page.objects.filter(project=base_project, listed=True, deleted=False).order_by('index') for task in tasks: new_task = Page(title=task.title, content=task.content, author=user, project=project) new_task.save() links = Link.objects.filter(project=base_project).order_by('index') for link in links: new_link = Link(name=link.name, url=link.url, user=user, project=project) new_link.save() project.create() messages.success(request, _('The %s has been cloned.') % project.kind.lower()) return http.HttpResponseRedirect(reverse('projects_create_tasks', kwargs={ 'slug': project.slug, })) else: messages.error(request, _("There was a problem cloning the study group, course, ...")) else: form = project_forms.CloneProjectForm() return render_to_response('projects/project_clone.html', { 'form': form, 'clone_tab': True, }, context_instance=RequestContext(request))
def test_link_attrs_validation(url): with pytest.raises(IncorrectUrl, match='Url is null or empty'): Link(url)
def test_domain_extracting_from_invalid_url(url): with pytest.raises(IncorrectUrl, match=f'Url "{url}" is incorrect'): Link(url).domain
def test_domain_extracting(url, exp): assert Link(url).domain == exp
def test_save(self): link = Link(link="https://example.com") link.save() self.assertIsNotNone(link.id)
def setUp(self): self.link_title = 'Книжный парад' self.link = Link(name=self.link_title)
def api_token(token_id): """ Manipulate a token. GET: Gets the current token data. Return: Object of {id, active, identity} POST: Change one or more of the token's data fields. The id cannot be set, but it can be forced to change, sending any value other than the current id, including null. Return: See GET. Error: If new identity would be a duplicate: 409 "Conflict" DELETE: Delete the token and all links owned by it. *Active session is required.* Returns: True PUT: Make new links owned by this token. Input: Array of strings (URLs) Return: Array of URL ids If a URL is invalid, the returned id will be null. """ token = Token.query.get(token_id) if token is None: raise NotFound('Token not found.') if request.method == 'POST': data = request.get_json() if data is None: raise BadRequest if 'active' in data and token.active != data['active']: token.active = data['active'] if 'identity' in data and token.identity != data['identity']: if Token.query.filter_by(owner=current_user, identity=data['identity']).limit(1).first(): raise Conflict('This identity already exists.') token.identity = data['identity'] if 'id' in data and token.id != data['id']: token.generate_id() db.session.add(token) db.session.commit() elif request.method == 'DELETE': if current_user.is_anonymous: raise Forbidden token.delete() return jsonify(True) elif request.method == 'PUT': data = request.get_json() links = [] for url in data: try: link = Link(token, url) links.append(link.id) db.session.add(link) except: links.append(None) db.session.commit() return jsonify(links) return jsonify({ 'id': token.id, 'active': token.active, 'identity': token.identity, })
def create_or_update_single_bill(self, data, pdf_link, link_file, gp=None): """ data - a dict of data for this gov proposal pdf_link - the source url from which the bill is taken link_file - a cached version of the pdf gp - an existing GovProposal objects. if this is given, it will be updated, instead of creating a new object """ if not (data['date']) or CUTOFF_DATE and data['date'] < CUTOFF_DATE: return law_name = data['law'] try: law, created = Law.objects.get_or_create(title=law_name) except Law.MultipleObjectsReturned: created = False try: law = Law.objects.filter(title=law_name, merged_into=None).last() except Law.MultipleObjectsReturned: # How is this possible? probably another bug somewhere law = Law.objects.filter(title=law_name).last() if created: law.save() if law.merged_into: law = law.merged_into title = u'' if data['correction']: title += data['correction'] if data['comment']: title += ' ' + data['comment'] if len(title) <= 1: title = u'חוק חדש' k_id = Knesset.objects.get_knesset_by_date(data['date']).pk if gp is None: # create new GovProposal, or look for an identical one (gp, created) = GovProposal.objects.get_or_create( booklet_number=data['booklet'], source_url=data['link'], title=title, law=law, date=data['date'], defaults={'knesset_id': k_id}) if created: gp.save() logger.debug("created GovProposal id = %d" % gp.id) # look for similar bills bill_params = dict(law=law, title=title, stage='3', stage_date=data['date']) similar_bills = Bill.objects.filter(**bill_params).order_by('id') if len(similar_bills) >= 1: b = similar_bills[0] if len(similar_bills) > 1: logger.debug("multiple bills detected") for bill in similar_bills: if bill.id == b.id: logger.debug("bill being used now: %d" % bill.id) else: logger.debug("bill with same fields: %d" % bill.id) else: # create a bill b = Bill(**bill_params) b.save() logger.debug("created bill %d" % b.id) # see if the found bill is already linked to a gov proposal try: bill_gp_id = b.gov_proposal.id except GovProposal.DoesNotExist: bill_gp_id = None if (bill_gp_id is None) or (gp.id == b.gov_proposal.id): # b is not linked to gp, or linked to the current gp gp.bill = b gp.save() else: logger.debug( "processing gp %d - matching bill (%d) already has gp" " (%d)" % (gp.id, b.id, b.gov_proposal.id)) else: # update a given GovProposal # TODO: move to a classmethod gp.booklet_number = data['booklet'] gp.knesset_id = k_id gp.source_url = data['link'] gp.title = title gp.law = law gp.date = data['date'] gp.save() gp.bill.title = title gp.bill.law = law gp.bill.save() b = gp.bill if (link_file is not None) and (link_file.link is None): link = Link(title=pdf_link, url=pdf_link, content_type=ContentType.objects.get_for_model(gp), object_pk=str(gp.id)) link.save() link_file.link = link link_file.save() logger.debug("check updated %s" % b.get_absolute_url())
def homeU(request): print("homeU called") if request.method == 'POST': if request.FILES['fileinput']: _user = request.user.is_authenticated if _user: print("user mode") print("request ", request.FILES['fileinput']) fileobject = FileObject() fileobject.name = str(request.FILES['fileinput']) fileobject.fileObject = request.FILES['fileinput'] datecur = timezone.datetime.now() fileobject.expiry = datetime(datecur.year + 100, datecur.month, datecur.day) if _user: fileobject.owner = request.user else: fileobject.owner = User.objects.get(username='******') fileobject.link = "" fileobject.save() _multi = request.POST.get('multiaccess', False) if _multi: fileobject.redirects = 9999 fileobject.link = "https://zurustorage.blob.core.windows.net/media/" + fileobject.fileObject.name fileobject.save() # create link for this file link = Link() link.to = fileobject.link link.is_fileobject = True if _user: if request.POST['fsuffixinput']: link.suffix, result = createSuffix( request.POST['fsuffixinput']) else: result = True link.suffix = createRandomSuffix() else: link.suffix = createRandomSuffix() result = True link.is_active = True if _multi: link.redirects = 9999 else: link.redirects = 1 datecur = timezone.datetime.now() link.expiry = datetime(datecur.year + 100, datecur.month, datecur.day) fileobject.suffix = link.suffix fileobject.save() if _user: link.owner = request.user user_account = Account.objects.get(username=request.user) user_account.linkset.append(link) user_account.fileset.append(fileobject) user_account.save() else: link.owner = User.objects.get(username='******') link.save() retlink = "zuru.io/" + link.suffix error = "Sorry! That URL is not available." if not result: return render(request, 'links/home.html', { 'retlink2': retlink, 'error': error }) else: return render(request, 'links/home.html', {'retlink2': retlink}) else: return render(request, 'links/home.html')
def clone(request): user = request.user.get_profile() if 'school' in request.GET: try: school = School.objects.get(slug=request.GET['school']) except School.DoesNotExist: return http.HttpResponseRedirect(reverse('projects_clone')) else: school = None if request.method == 'POST': form = project_forms.CloneProjectForm(school, request.POST) if form.is_valid(): base_project = form.cleaned_data['project'] project = Project(name=base_project.name, short_description=base_project.short_description, long_description=base_project.long_description, school=base_project.school, clone_of=base_project) project.save() act = Activity(actor=user, verb='http://activitystrea.ms/schema/1.0/post', project=project, target_project=project) act.save() participation = Participation(project=project, user=user, organizing=True) participation.save() new_rel = Relationship(source=user, target_project=project) try: new_rel.save() except IntegrityError: pass detailed_description = Page(title=_('Full Description'), slug='full-description', content=base_project.detailed_description.content, listed=False, author_id=user.id, project_id=project.id) detailed_description.save() project.detailed_description_id = detailed_description.id sign_up = Page(title=_('Sign-Up'), slug='sign-up', content=base_project.sign_up.content, listed=False, editable=False, author_id=user.id, project_id=project.id) sign_up.save() project.sign_up_id = sign_up.id project.save() tasks = Page.objects.filter(project=base_project, listed=True, deleted=False).order_by('index') for task in tasks: new_task = Page(title=task.title, content=task.content, author=user, project=project) new_task.save() links = Link.objects.filter(project=base_project).order_by('index') for link in links: new_link = Link(name=link.name, url=link.url, user=user, project=project) new_link.save() messages.success(request, _('The study group has been cloned.')) return http.HttpResponseRedirect(reverse('projects_show', kwargs={ 'slug': project.slug, })) else: messages.error(request, _("There was a problem cloning the study group.")) else: form = project_forms.CloneProjectForm(school) return render_to_response('projects/project_clone.html', { 'form': form, 'clone_tab': True, 'school': school, }, context_instance=RequestContext(request))
def import_from_old_site(request): user = request.user.get_profile() if 'school' in request.GET: try: school = School.objects.get(slug=request.GET['school']) except School.DoesNotExist: return http.HttpResponseRedirect(reverse('projects_clone')) else: school = None if request.method == 'POST': form = project_forms.ImportProjectForm(school, request.POST) if form.is_valid(): course = form.cleaned_data['course'] project = Project(name=course['name'], short_description= course['short_description'], long_description=course['long_description'], school=course['school'], imported_from=course['slug']) project.save() act = Activity(actor=user, verb='http://activitystrea.ms/schema/1.0/post', project=project, target_project=project) act.save() participation = Participation(project=project, user=user, organizing=True) participation.save() new_rel = Relationship(source=user, target_project=project) try: new_rel.save() except IntegrityError: pass if course['detailed_description']: detailed_description_content = course['detailed_description'] else: detailed_description_content = render_to_string( "projects/detailed_description_initial_content.html", {}) detailed_description = Page(title=_('Full Description'), slug='full-description', content=detailed_description_content, listed=False, author_id=user.id, project_id=project.id) detailed_description.save() project.detailed_description_id = detailed_description.id sign_up_content = render_to_string("projects/sign_up_initial_content.html", {}) sign_up = Page(title=_('Sign-Up'), slug='sign-up', content=sign_up_content, listed=False, editable=False, author_id=user.id, project_id=project.id) sign_up.save() project.sign_up_id = sign_up.id project.save() for title, content in course['tasks']: new_task = Page(title=title, content=content, author=user, project=project) new_task.save() for name, url in course['links']: new_link = Link(name=name, url=url, user=user, project=project) new_link.save() messages.success(request, _('The study group has been imported.')) return http.HttpResponseRedirect(reverse('projects_show', kwargs={ 'slug': project.slug, })) else: messages.error(request, _("There was a problem importing the study group.")) else: form = project_forms.ImportProjectForm(school) return render_to_response('projects/project_import.html', { 'form': form, 'import_tab': True, 'school': school, }, context_instance=RequestContext(request))
def update_db_from_files(self): logger.debug("Update DB From Files") try: laws = [] # of lists: [name,name_for_search,explanation,link] f = gzip.open(os.path.join(DATA_ROOT, 'laws.tsv.gz')) content = f.read().split('\n') for line in content: law = line.split('\t') if len(law) == 3: name_for_search = self.get_search_string(law[0]) law.insert(1, name_for_search) laws.append(law) f.close() parties = dict() # key: party-name; value: Party members = dict() # key: member-name; value: Member votes = dict() # key: id; value: Vote memberships = dict() # key: (member.id,party.id) current_max_src_id = Vote.objects.aggregate(Max('src_id'))['src_id__max'] if current_max_src_id == None: # the db contains no votes, meaning its empty current_max_src_id = 0 logger.debug("processing votes data") f = gzip.open(os.path.join(DATA_ROOT, 'votes.tsv.gz')) content = f.read().split('\n') for line in content: if len(line) <= 1: continue (vote_id, vote_src_url, vote_label, vote_meeting_num, vote_num, vote_time_string, _, _, _, _) = line.split('\t') # if vote_id < current_max_src_id: # skip votes already parsed. # continue vote_time_string = vote_time_string.replace(' ', ' ') for i in self.heb_months: if i in vote_time_string: month = self.heb_months.index(i) + 1 day = re.search("""(\d\d?)""", vote_time_string).group(1) year = re.search("""(\d\d\d\d)""", vote_time_string).group(1) vote_hm = datetime.datetime.strptime(vote_time_string.split(' ')[-1], "%H:%M") vote_date = datetime.date(int(year), int(month), int(day)) vote_time = datetime.datetime(int(year), int(month), int(day), vote_hm.hour, vote_hm.minute) vote_label_for_search = self.get_search_string(vote_label) # if vote_date < datetime.date(2009, 02, 24): # vote before 18th knesset # continue try: v = Vote.objects.get(src_id=vote_id) created = False except: v = Vote(title=vote_label, time_string=vote_time_string, importance=1, src_id=vote_id, time=vote_time) try: vote_meeting_num = int(vote_meeting_num) v.meeting_number = vote_meeting_num except: pass try: vote_num = int(vote_num) v.vote_number = vote_num except: pass v.src_url = vote_src_url for law in laws: (_, law_name_for_search, law_exp, law_link) = law if vote_label_for_search.find(law_name_for_search) >= 0: v.summary = law_exp v.full_text_url = law_link v.save() if v.full_text_url != None: l = Link(title=u'מסמך הצעת החוק באתר הכנסת', url=v.full_text_url, content_type=ContentType.objects.get_for_model(v), object_pk=str(v.id)) l.save() votes[int(vote_id)] = v f.close() logger.debug("processing member votes data") f = gzip.open(os.path.join(DATA_ROOT, 'results.tsv.gz')) content = f.read().split('\n') for line in content: if len(line) < 2: continue s = line.split('\t') # (id,voter,party,vote) vote_id = int(s[0]) voter = s[1] voter_party = s[2] # transform party names to canonical form if voter_party in CANONICAL_PARTY_ALIASES: voter_party = CANONICAL_PARTY_ALIASES[voter_party] vote = s[3] try: v = votes[vote_id] except KeyError: # this vote was skipped in this read, also skip voteactions and members continue vote_date = v.time.date() # create/get the party appearing in this vote if voter_party in parties: party = parties[voter_party] created = False else: party, created = Party.objects.get_or_create(name=voter_party) parties[voter_party] = party # if created: # this is magic needed because of unicode chars. if you don't do this, the object p will have gibrish as its name. # only when it comes back from the db it has valid unicode chars. # use this vote's time to update the party's start date and end date if (party.start_date is None) or (party.start_date > vote_date): party.start_date = vote_date if (party.end_date is None) or (party.end_date < vote_date): party.end_date = vote_date if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end party.save() # create/get the member voting if voter in members: member = members[voter] created = False else: try: member = Member.objects.get(name=voter) except: # if there are several people with same age, member = Member.objects.filter(name=voter).order_by('-date_of_birth')[ 0] # choose the younger. TODO: fix this members[voter] = member # use this vote's date to update the member's dates. if (member.start_date is None) or (member.start_date > vote_date): member.start_date = vote_date if (member.end_date is None) or (member.end_date < vote_date): member.end_date = vote_date # if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end # m.save() # create/get the membership (connection between member and party) if ((member.id, party.id) in memberships): ms = memberships[(member.id, party.id)] created = False else: ms, created = Membership.objects.get_or_create(member=member, party=party) memberships[(member.id, party.id)] = ms # if created: # again, unicode magic # ms = Membership.objects.get(member=m,party=p) # again, update the dates on the membership if (ms.start_date is None) or (ms.start_date > vote_date): ms.start_date = vote_date if (ms.end_date is None) or (ms.end_date < vote_date): ms.end_date = vote_date if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end ms.save() # add the current member's vote va, created = VoteAction.objects.get_or_create(vote=v, member=member, type=vote, party=member.current_party) if created: va.save() logger.debug("done") logger.debug( "saving data: %d parties, %d members, %d memberships " % (len(parties), len(members), len(memberships))) for party in parties: parties[party].save() for member in members: members[member].save() for ms in memberships: memberships[ms].save() logger.debug("done") f.close() except Exception: logger.exception('Update db from file exception')
def update_members_from_file(self): logger.debug('update_members_from_file') f = gzip.open(os.path.join(DATA_ROOT, 'members.tsv.gz')) content = f.read().split('\n') for line in content: if len(line) <= 1: continue (member_id, name, img_url, phone, fax, website, email, family_status, number_of_children, date_of_birth, year_of_birth, place_of_birth, date_of_death, year_of_aliyah, k18, k19, _) = line.split('\t') if email != '': email = email.split(':')[1] try: if date_of_birth.find(',') >= 0: date_of_birth = date_of_birth.split(',')[1].strip(' ') date_of_birth = datetime.datetime.strptime(date_of_birth, "%d/%m/%Y") except: date_of_birth = None try: if date_of_birth.find(',') >= 0: date_of_death = date_of_birth.split(',')[1].strip(' ') date_of_death = datetime.datetime.strptime(date_of_death, "%d/%m/%Y") except: date_of_death = None try: year_of_birth = int(year_of_birth) except: year_of_birth = None try: year_of_aliyah = int(year_of_aliyah) except: year_of_aliyah = None try: number_of_children = int(number_of_children) except: number_of_children = None try: m = Member.objects.get(id=member_id) m.phone = phone m.fax = fax m.email = email m.family_status = family_status m.number_of_children = number_of_children m.date_of_death = date_of_death m.save() logger.debug('updated member %d' % m.id) except Member.DoesNotExist: # member_id not found. create new m = Member(id=member_id, name=name, img_url=img_url, phone=phone, fax=fax, website=None, email=email, family_status=family_status, number_of_children=number_of_children, date_of_birth=date_of_birth, place_of_birth=place_of_birth, date_of_death=date_of_death, year_of_aliyah=year_of_aliyah) m.save() m = Member.objects.get(pk=member_id) # make sure we are are # working on the db object. e.g m.id is a number. logger.debug('created member %d' % m.id) if len(website) > 0: l = Link(title='אתר האינטרנט של %s' % name, url=website, content_type=ContentType.objects.get_for_model(m), object_pk=str(m.id)) l.save() if k19: # KNESSET 19 specific parties = Party.objects.filter(knesset_id=19).values_list('name', 'id') k19 = k19.decode(ENCODING) for k, v in parties: if k in k19: m.current_party_id = int(v) logger.debug('member %s, k19 %s, party %s' % (m.name, k19, k)) m.save() if m.current_party is None: logger.debug('member %s, k19 %s not found' % (m.name, k19))