def test_entity_description(self):
link_to_school = Link(
destination_content_type=ContentType.objects.get_for_model(
self.school
),
destination_object_id=self.school.id,
)
self.assertEquals(link_to_school.__unicode__(), self.school.name)
link_to_building = Link(
destination_content_type = ContentType.objects.get_for_model(self.main_building),
destination_object_id = self.main_building.id,
)
self.assertEquals(
link_to_building.__unicode__(),
self.main_building.__unicode__()
)
self.assertEquals(
link_to_building.summary(),
u"St Mary's Street, Cardiff"
)
"""
def import_from_old_site(request):
user = request.user.get_profile()
if request.method == 'POST':
form = project_forms.ImportProjectForm(request.POST)
if form.is_valid():
course = form.cleaned_data['course']
#CS - too much logic in view
project = Project(name=course['name'], kind=course['kind'],
short_description=course['short_description'],
long_description=course['long_description'],
imported_from=course['slug'])
project.save()
act = Activity(actor=user,
verb=verbs['post'],
scope_object=project,
target_object=project)
act.save()
participation = Participation(project=project, user=user,
organizing=True)
participation.save()
new_rel, created = Relationship.objects.get_or_create(source=user,
target_project=project)
new_rel.deleted = False
new_rel.save()
if course['detailed_description']:
detailed_description_content = course['detailed_description']
else:
detailed_description_content = render_to_string(
"projects/detailed_description_initial_content.html",
{'project': project})
detailed_description = Page(title=_('Full Description'),
slug='full-description', content=detailed_description_content,
listed=False, author_id=user.id, project_id=project.id)
detailed_description.save()
project.detailed_description_id = detailed_description.id
sign_up = Signup(between_participants=course['sign_up'],
author_id=user.id, project_id=project.id)
sign_up.save()
project.save()
for title, content in course['tasks']:
new_task = Page(title=title, content=content, author=user,
project=project)
new_task.save()
for name, url in course['links']:
new_link = Link(name=name, url=url, user=user, project=project)
new_link.save()
project.create()
messages.success(request,
_('The %s has been imported.') % project.kind.lower())
return http.HttpResponseRedirect(reverse('projects_show', kwargs={
'slug': project.slug,
}))
else:
msg = _("Problem importing the study group, course, ...")
messages.error(request, msg)
else:
form = project_forms.ImportProjectForm()
return render_to_response('projects/project_import.html', {
'form': form, 'import_tab': True},
context_instance=RequestContext(request))
def mutate(self, info, url, description):
link = Link(url=url, description=description)
link.save()
return CreateLink(id=link.id,
url=link.url,
description=link.description)
def mutate_and_get_payload(root, info, **input):
user = info.context.user or None
link = Link(url=input.get('url'),
description=input.get('description'),
posted_by=user)
link.save()
return RelayCreateLink(link=link)
def mutate(self, info, url, description):
user = info.context.user or None
link = Link(url=url, description=description, posted_by=user)
link.save()
return CreateLink(id=link.id,
url=link.url,
description=link.description,
posted_by=link.posted_by)
def mutate(self, info, url, description):
user = get_user(info) or None
link = Link(url=url, description=description, posted_by=user)
## save the date
link.save()
return CreateLink(id=link.id,
url=link.url,
description=link.description,
posted_by=link.posted_by)
def mutate_and_get_payload(self, info, **input):
user = info.context.user or None
logger.info('12345 上山打老虎')
link = Link(
url=input.get('url'),
description=input.get('description'),
posted_by=user,
)
link.save()
return RelayCreateLink(link=link)
def mutate(self, info, url, description):
# will fail if there is no logged in user
posted_by = get_user(info)
link = Link(url=url, description=description, posted_by=posted_by)
link.save()
return CreateLink(
id=link.id,
url=link.url,
description=link.description,
posted_by=link.posted_by,
)
def test_link_encode(self):
tests = [
(1, 'MQ=='),
(100, 'MTAw'),
(10000, 'MTAwMDA='),
(1000000, 'MTAwMDAwMA=='),
]
for test in tests:
l = Link(id=test[0], url='http://example.com')
key = l.encode()
self.assertEqual(key, test[1])
def home(request):
print("home called")
if request.method == 'POST':
if request.POST['urlinput']:
_user = request.user.is_authenticated
if _user:
print("user-mode")
print("request ", request.POST['urlinput'])
link = Link()
link.to = request.POST['urlinput']
result = False
if _user:
if request.POST['suffixinput']:
link.suffix, result = createSuffix(
request.POST['suffixinput'])
else:
result = True
link.suffix = createRandomSuffix()
else:
result = True
link.suffix = createRandomSuffix()
link.is_active = True
is_onetime_checkbox = request.POST.get('onetimeinput', False)
if is_onetime_checkbox:
link.redirects = 1
datecur = timezone.datetime.now()
link.expiry = datetime(datecur.year + 100, datecur.month,
datecur.day)
if _user:
link.owner = request.user
user_account = Account.objects.get(username=request.user)
user_account.linkset.append(link)
user_account.save()
else:
link.owner = User.objects.get(username='******')
link.save()
retlink = "zuru.io/" + link.suffix
error = "Sorry! That URL is not available."
if not result:
return render(request, 'links/home.html', {
'retlink1': retlink,
'error': error
})
else:
return render(request, 'links/home.html',
{'retlink1': retlink})
else:
return render(request, 'links/home.html')
def edit_url(request: HttpRequest, name: str, context: dict):
link = None
if name is not None:
try:
link = Link.objects.get(name=name)
except Exception:
raise Http404
context['link'] = link
if request.POST:
formname = request.POST['name']
url = request.POST['url']
try:
request.POST['instant']
instant = True
except:
instant = False
nullcheck = assure_formname_and_url_not_null(formname, url, request,
context)
if (nullcheck):
return nullcheck
del nullcheck
formname = formname.replace(" ", "_")
name_in_use = assure_formname_is_not_in_use(formname, name, request,
context)
if (name_in_use):
return name_in_use
del name_in_use
try:
URLValidator(None).__call__(url)
except ValidationError:
context = dangeralert(context, 'Thats not a valid URL!')
return render(request, 'dashlink.html', context)
try:
if link is None:
link = Link(url=url, name=formname, instant_redirect=instant)
else:
link.url = url
link.name = formname
link.instant_redirect = instant
link.save()
except Exception as e:
print(e)
context = dangeralert(context, 'An error has occured!')
return render(request, 'dashlink.html', context)
return HttpResponseRedirect(reverse('dashboard:index'))
return HttpResponse("You can only POST this page.")
def token(token):
token = Token.query.filter_by(owner=current_user, identity=token).limit(1).first()
if not token:
raise NotFound('Token not found.')
form = AddLinkForm()
if form.submit.data:
if form.validate_on_submit():
link = Link(token, form.url.data)
db.session.add(link)
db.session.commit()
# Clears form and allows page refresh without re-submitting the form.
return redirect(url_for('token', token=token.identity))
for k, v in form.errors.items():
for m in v:
flash(m, 'danger')
return render_template('token.html', token=token, form=form)
def create_or_update_single_bill(self, proposal, pdf_link, link_file):
if not(proposal['date']) or CUTOFF_DATE and proposal['date'] < CUTOFF_DATE:
return
law_name = proposal['law']
(law, created) = Law.objects.get_or_create(title=law_name)
if created:
law.save()
if law.merged_into:
law = law.merged_into
title = u''
if proposal['correction']:
title += proposal['correction']
if proposal['comment']:
title += ' ' + proposal['comment']
if len(title)<=1:
title = u'חוק חדש'
(gp,created) = GovProposal.objects.get_or_create(booklet_number=proposal['booklet'], knesset_id=18,
source_url=proposal['link'],
title=title, law=law, date=proposal['date'])
if created:
gp.save()
logger.debug("created GovProposal id = %d" % gp.id)
bill_params = dict(law=law, title=title, stage='3', stage_date=proposal['date'])
similar_bills = Bill.objects.filter(**bill_params).order_by('id')
if len(similar_bills) >= 1:
b = similar_bills[0]
if len(similar_bills) > 1:
logger.debug("multiple bills detected")
for bill in similar_bills:
if bill.id == b.id:
logger.debug("bill being used now - %d" % bill.id)
else:
logger.debug("bill with same fields - %d" % bill.id)
else:
b = Bill(**bill_params)
b.save()
gp.bill = b
gp.save()
if link_file.link is None:
link = Link(title=pdf_link, url=pdf_link,
content_type=ContentType.objects.get_for_model(gp),
object_pk=str(gp.id))
link.save()
link_file.link = link
link_file.save()
logger.debug("check updated %s" % b.get_absolute_url())
def mutate(root, input, context, info):
user = get_user(context) or None
link = Link(
url=input.get('url'),
description=input.get('description'),
posted_by=user,
)
link.save()
return CreateLink(
id=link.id,
url=link.url,
description=link.description,
posted_by=link.posted_by,
)
def clone(request):
user = request.user.get_profile()
if request.method == 'POST':
form = project_forms.CloneProjectForm(request.POST)
if form.is_valid():
base_project = form.cleaned_data['project']
#CS - too much logic in view
project = Project(name=base_project.name,
category=base_project.category,
other=base_project.other,
other_description=base_project.other_description,
short_description=base_project.short_description,
long_description=base_project.long_description,
clone_of=base_project)
project.save()
act = Activity(actor=user,
verb=verbs['post'],
scope_object=project,
target_object=project)
act.save()
participation = Participation(project=project, user=user,
organizing=True)
participation.save()
new_rel, created = Relationship.objects.get_or_create(source=user,
target_project=project)
new_rel.deleted = False
new_rel.save()
detailed_description = Page(title=_('Full Description'),
slug='full-description',
content=base_project.detailed_description.content,
listed=False, author_id=user.id, project_id=project.id)
detailed_description.save()
project.detailed_description_id = detailed_description.id
base_sign_up = base_project.sign_up.get()
sign_up = Signup(public=base_sign_up.public,
between_participants=base_sign_up.between_participants,
author_id=user.id, project_id=project.id)
sign_up.save()
project.save()
tasks = Page.objects.filter(project=base_project, listed=True,
deleted=False).order_by('index')
for task in tasks:
new_task = Page(title=task.title, content=task.content,
author=user, project=project)
new_task.save()
links = Link.objects.filter(project=base_project).order_by('index')
for link in links:
new_link = Link(name=link.name, url=link.url, user=user,
project=project)
new_link.save()
project.create()
messages.success(request,
_('The %s has been cloned.') % project.kind.lower())
return http.HttpResponseRedirect(reverse('projects_create_tasks', kwargs={
'slug': project.slug,
}))
else:
messages.error(request,
_("There was a problem cloning the study group, course, ..."))
else:
form = project_forms.CloneProjectForm()
return render_to_response('projects/project_clone.html', {
'form': form, 'clone_tab': True,
}, context_instance=RequestContext(request))
def test_link_attrs_validation(url):
with pytest.raises(IncorrectUrl, match='Url is null or empty'):
Link(url)
def test_domain_extracting_from_invalid_url(url):
with pytest.raises(IncorrectUrl, match=f'Url "{url}" is incorrect'):
Link(url).domain
def test_domain_extracting(url, exp):
assert Link(url).domain == exp
def test_save(self):
link = Link(link="https://example.com")
link.save()
self.assertIsNotNone(link.id)
def setUp(self):
self.link_title = 'Книжный парад'
self.link = Link(name=self.link_title)
def api_token(token_id):
"""
Manipulate a token.
GET:
Gets the current token data.
Return:
Object of {id, active, identity}
POST:
Change one or more of the token's data fields.
The id cannot be set, but it can be forced to change, sending any value other than the current id, including null.
Return:
See GET.
Error:
If new identity would be a duplicate:
409 "Conflict"
DELETE:
Delete the token and all links owned by it.
*Active session is required.*
Returns:
True
PUT:
Make new links owned by this token.
Input:
Array of strings (URLs)
Return:
Array of URL ids
If a URL is invalid, the returned id will be null.
"""
token = Token.query.get(token_id)
if token is None:
raise NotFound('Token not found.')
if request.method == 'POST':
data = request.get_json()
if data is None:
raise BadRequest
if 'active' in data and token.active != data['active']:
token.active = data['active']
if 'identity' in data and token.identity != data['identity']:
if Token.query.filter_by(owner=current_user, identity=data['identity']).limit(1).first():
raise Conflict('This identity already exists.')
token.identity = data['identity']
if 'id' in data and token.id != data['id']:
token.generate_id()
db.session.add(token)
db.session.commit()
elif request.method == 'DELETE':
if current_user.is_anonymous:
raise Forbidden
token.delete()
return jsonify(True)
elif request.method == 'PUT':
data = request.get_json()
links = []
for url in data:
try:
link = Link(token, url)
links.append(link.id)
db.session.add(link)
except:
links.append(None)
db.session.commit()
return jsonify(links)
return jsonify({
'id': token.id,
'active': token.active,
'identity': token.identity,
})
def create_or_update_single_bill(self, data, pdf_link, link_file, gp=None):
"""
data - a dict of data for this gov proposal
pdf_link - the source url from which the bill is taken
link_file - a cached version of the pdf
gp - an existing GovProposal objects. if this is given, it will be
updated, instead of creating a new object
"""
if not (data['date']) or CUTOFF_DATE and data['date'] < CUTOFF_DATE:
return
law_name = data['law']
try:
law, created = Law.objects.get_or_create(title=law_name)
except Law.MultipleObjectsReturned:
created = False
try:
law = Law.objects.filter(title=law_name,
merged_into=None).last()
except Law.MultipleObjectsReturned: # How is this possible? probably another bug somewhere
law = Law.objects.filter(title=law_name).last()
if created:
law.save()
if law.merged_into:
law = law.merged_into
title = u''
if data['correction']:
title += data['correction']
if data['comment']:
title += ' ' + data['comment']
if len(title) <= 1:
title = u'חוק חדש'
k_id = Knesset.objects.get_knesset_by_date(data['date']).pk
if gp is None: # create new GovProposal, or look for an identical one
(gp, created) = GovProposal.objects.get_or_create(
booklet_number=data['booklet'],
source_url=data['link'],
title=title,
law=law,
date=data['date'],
defaults={'knesset_id': k_id})
if created:
gp.save()
logger.debug("created GovProposal id = %d" % gp.id)
# look for similar bills
bill_params = dict(law=law,
title=title,
stage='3',
stage_date=data['date'])
similar_bills = Bill.objects.filter(**bill_params).order_by('id')
if len(similar_bills) >= 1:
b = similar_bills[0]
if len(similar_bills) > 1:
logger.debug("multiple bills detected")
for bill in similar_bills:
if bill.id == b.id:
logger.debug("bill being used now: %d" % bill.id)
else:
logger.debug("bill with same fields: %d" % bill.id)
else: # create a bill
b = Bill(**bill_params)
b.save()
logger.debug("created bill %d" % b.id)
# see if the found bill is already linked to a gov proposal
try:
bill_gp_id = b.gov_proposal.id
except GovProposal.DoesNotExist:
bill_gp_id = None
if (bill_gp_id is None) or (gp.id == b.gov_proposal.id):
# b is not linked to gp, or linked to the current gp
gp.bill = b
gp.save()
else:
logger.debug(
"processing gp %d - matching bill (%d) already has gp"
" (%d)" % (gp.id, b.id, b.gov_proposal.id))
else: # update a given GovProposal
# TODO: move to a classmethod
gp.booklet_number = data['booklet']
gp.knesset_id = k_id
gp.source_url = data['link']
gp.title = title
gp.law = law
gp.date = data['date']
gp.save()
gp.bill.title = title
gp.bill.law = law
gp.bill.save()
b = gp.bill
if (link_file is not None) and (link_file.link is None):
link = Link(title=pdf_link,
url=pdf_link,
content_type=ContentType.objects.get_for_model(gp),
object_pk=str(gp.id))
link.save()
link_file.link = link
link_file.save()
logger.debug("check updated %s" % b.get_absolute_url())
def homeU(request):
print("homeU called")
if request.method == 'POST':
if request.FILES['fileinput']:
_user = request.user.is_authenticated
if _user:
print("user mode")
print("request ", request.FILES['fileinput'])
fileobject = FileObject()
fileobject.name = str(request.FILES['fileinput'])
fileobject.fileObject = request.FILES['fileinput']
datecur = timezone.datetime.now()
fileobject.expiry = datetime(datecur.year + 100, datecur.month,
datecur.day)
if _user:
fileobject.owner = request.user
else:
fileobject.owner = User.objects.get(username='******')
fileobject.link = ""
fileobject.save()
_multi = request.POST.get('multiaccess', False)
if _multi:
fileobject.redirects = 9999
fileobject.link = "https://zurustorage.blob.core.windows.net/media/" + fileobject.fileObject.name
fileobject.save()
# create link for this file
link = Link()
link.to = fileobject.link
link.is_fileobject = True
if _user:
if request.POST['fsuffixinput']:
link.suffix, result = createSuffix(
request.POST['fsuffixinput'])
else:
result = True
link.suffix = createRandomSuffix()
else:
link.suffix = createRandomSuffix()
result = True
link.is_active = True
if _multi:
link.redirects = 9999
else:
link.redirects = 1
datecur = timezone.datetime.now()
link.expiry = datetime(datecur.year + 100, datecur.month,
datecur.day)
fileobject.suffix = link.suffix
fileobject.save()
if _user:
link.owner = request.user
user_account = Account.objects.get(username=request.user)
user_account.linkset.append(link)
user_account.fileset.append(fileobject)
user_account.save()
else:
link.owner = User.objects.get(username='******')
link.save()
retlink = "zuru.io/" + link.suffix
error = "Sorry! That URL is not available."
if not result:
return render(request, 'links/home.html', {
'retlink2': retlink,
'error': error
})
else:
return render(request, 'links/home.html',
{'retlink2': retlink})
else:
return render(request, 'links/home.html')
def clone(request):
user = request.user.get_profile()
if 'school' in request.GET:
try:
school = School.objects.get(slug=request.GET['school'])
except School.DoesNotExist:
return http.HttpResponseRedirect(reverse('projects_clone'))
else:
school = None
if request.method == 'POST':
form = project_forms.CloneProjectForm(school, request.POST)
if form.is_valid():
base_project = form.cleaned_data['project']
project = Project(name=base_project.name,
short_description=base_project.short_description,
long_description=base_project.long_description,
school=base_project.school, clone_of=base_project)
project.save()
act = Activity(actor=user,
verb='http://activitystrea.ms/schema/1.0/post',
project=project,
target_project=project)
act.save()
participation = Participation(project=project, user=user, organizing=True)
participation.save()
new_rel = Relationship(source=user, target_project=project)
try:
new_rel.save()
except IntegrityError:
pass
detailed_description = Page(title=_('Full Description'), slug='full-description',
content=base_project.detailed_description.content, listed=False,
author_id=user.id, project_id=project.id)
detailed_description.save()
project.detailed_description_id = detailed_description.id
sign_up = Page(title=_('Sign-Up'), slug='sign-up',
content=base_project.sign_up.content, listed=False, editable=False,
author_id=user.id, project_id=project.id)
sign_up.save()
project.sign_up_id = sign_up.id
project.save()
tasks = Page.objects.filter(project=base_project, listed=True,
deleted=False).order_by('index')
for task in tasks:
new_task = Page(title=task.title, content=task.content, author=user,
project=project)
new_task.save()
links = Link.objects.filter(project=base_project).order_by('index')
for link in links:
new_link = Link(name=link.name, url=link.url, user=user, project=project)
new_link.save()
messages.success(request, _('The study group has been cloned.'))
return http.HttpResponseRedirect(reverse('projects_show', kwargs={
'slug': project.slug,
}))
else:
messages.error(request,
_("There was a problem cloning the study group."))
else:
form = project_forms.CloneProjectForm(school)
return render_to_response('projects/project_clone.html', {
'form': form, 'clone_tab': True, 'school': school,
}, context_instance=RequestContext(request))
def import_from_old_site(request):
user = request.user.get_profile()
if 'school' in request.GET:
try:
school = School.objects.get(slug=request.GET['school'])
except School.DoesNotExist:
return http.HttpResponseRedirect(reverse('projects_clone'))
else:
school = None
if request.method == 'POST':
form = project_forms.ImportProjectForm(school, request.POST)
if form.is_valid():
course = form.cleaned_data['course']
project = Project(name=course['name'],
short_description= course['short_description'],
long_description=course['long_description'],
school=course['school'], imported_from=course['slug'])
project.save()
act = Activity(actor=user,
verb='http://activitystrea.ms/schema/1.0/post',
project=project,
target_project=project)
act.save()
participation = Participation(project=project, user=user, organizing=True)
participation.save()
new_rel = Relationship(source=user, target_project=project)
try:
new_rel.save()
except IntegrityError:
pass
if course['detailed_description']:
detailed_description_content = course['detailed_description']
else:
detailed_description_content = render_to_string(
"projects/detailed_description_initial_content.html",
{})
detailed_description = Page(title=_('Full Description'), slug='full-description',
content=detailed_description_content, listed=False,
author_id=user.id, project_id=project.id)
detailed_description.save()
project.detailed_description_id = detailed_description.id
sign_up_content = render_to_string("projects/sign_up_initial_content.html",
{})
sign_up = Page(title=_('Sign-Up'), slug='sign-up',
content=sign_up_content, listed=False, editable=False,
author_id=user.id, project_id=project.id)
sign_up.save()
project.sign_up_id = sign_up.id
project.save()
for title, content in course['tasks']:
new_task = Page(title=title, content=content, author=user,
project=project)
new_task.save()
for name, url in course['links']:
new_link = Link(name=name, url=url, user=user, project=project)
new_link.save()
messages.success(request, _('The study group has been imported.'))
return http.HttpResponseRedirect(reverse('projects_show', kwargs={
'slug': project.slug,
}))
else:
messages.error(request,
_("There was a problem importing the study group."))
else:
form = project_forms.ImportProjectForm(school)
return render_to_response('projects/project_import.html', {
'form': form, 'import_tab': True, 'school': school,
}, context_instance=RequestContext(request))
def update_db_from_files(self):
logger.debug("Update DB From Files")
try:
laws = [] # of lists: [name,name_for_search,explanation,link]
f = gzip.open(os.path.join(DATA_ROOT, 'laws.tsv.gz'))
content = f.read().split('\n')
for line in content:
law = line.split('\t')
if len(law) == 3:
name_for_search = self.get_search_string(law[0])
law.insert(1, name_for_search)
laws.append(law)
f.close()
parties = dict() # key: party-name; value: Party
members = dict() # key: member-name; value: Member
votes = dict() # key: id; value: Vote
memberships = dict() # key: (member.id,party.id)
current_max_src_id = Vote.objects.aggregate(Max('src_id'))['src_id__max']
if current_max_src_id == None: # the db contains no votes, meaning its empty
current_max_src_id = 0
logger.debug("processing votes data")
f = gzip.open(os.path.join(DATA_ROOT, 'votes.tsv.gz'))
content = f.read().split('\n')
for line in content:
if len(line) <= 1:
continue
(vote_id, vote_src_url, vote_label, vote_meeting_num, vote_num, vote_time_string, _, _, _,
_) = line.split('\t')
# if vote_id < current_max_src_id: # skip votes already parsed.
# continue
vote_time_string = vote_time_string.replace(' ', ' ')
for i in self.heb_months:
if i in vote_time_string:
month = self.heb_months.index(i) + 1
day = re.search("""(\d\d?)""", vote_time_string).group(1)
year = re.search("""(\d\d\d\d)""", vote_time_string).group(1)
vote_hm = datetime.datetime.strptime(vote_time_string.split(' ')[-1], "%H:%M")
vote_date = datetime.date(int(year), int(month), int(day))
vote_time = datetime.datetime(int(year), int(month), int(day), vote_hm.hour, vote_hm.minute)
vote_label_for_search = self.get_search_string(vote_label)
# if vote_date < datetime.date(2009, 02, 24): # vote before 18th knesset
# continue
try:
v = Vote.objects.get(src_id=vote_id)
created = False
except:
v = Vote(title=vote_label, time_string=vote_time_string, importance=1, src_id=vote_id,
time=vote_time)
try:
vote_meeting_num = int(vote_meeting_num)
v.meeting_number = vote_meeting_num
except:
pass
try:
vote_num = int(vote_num)
v.vote_number = vote_num
except:
pass
v.src_url = vote_src_url
for law in laws:
(_, law_name_for_search, law_exp, law_link) = law
if vote_label_for_search.find(law_name_for_search) >= 0:
v.summary = law_exp
v.full_text_url = law_link
v.save()
if v.full_text_url != None:
l = Link(title=u'מסמך הצעת החוק באתר הכנסת', url=v.full_text_url,
content_type=ContentType.objects.get_for_model(v), object_pk=str(v.id))
l.save()
votes[int(vote_id)] = v
f.close()
logger.debug("processing member votes data")
f = gzip.open(os.path.join(DATA_ROOT, 'results.tsv.gz'))
content = f.read().split('\n')
for line in content:
if len(line) < 2:
continue
s = line.split('\t') # (id,voter,party,vote)
vote_id = int(s[0])
voter = s[1]
voter_party = s[2]
# transform party names to canonical form
if voter_party in CANONICAL_PARTY_ALIASES:
voter_party = CANONICAL_PARTY_ALIASES[voter_party]
vote = s[3]
try:
v = votes[vote_id]
except KeyError: # this vote was skipped in this read, also skip voteactions and members
continue
vote_date = v.time.date()
# create/get the party appearing in this vote
if voter_party in parties:
party = parties[voter_party]
created = False
else:
party, created = Party.objects.get_or_create(name=voter_party)
parties[voter_party] = party
# if created: # this is magic needed because of unicode chars. if you don't do this, the object p will have gibrish as its name.
# only when it comes back from the db it has valid unicode chars.
# use this vote's time to update the party's start date and end date
if (party.start_date is None) or (party.start_date > vote_date):
party.start_date = vote_date
if (party.end_date is None) or (party.end_date < vote_date):
party.end_date = vote_date
if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end
party.save()
# create/get the member voting
if voter in members:
member = members[voter]
created = False
else:
try:
member = Member.objects.get(name=voter)
except: # if there are several people with same age,
member = Member.objects.filter(name=voter).order_by('-date_of_birth')[
0] # choose the younger. TODO: fix this
members[voter] = member
# use this vote's date to update the member's dates.
if (member.start_date is None) or (member.start_date > vote_date):
member.start_date = vote_date
if (member.end_date is None) or (member.end_date < vote_date):
member.end_date = vote_date
# if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end
# m.save()
# create/get the membership (connection between member and party)
if ((member.id, party.id) in memberships):
ms = memberships[(member.id, party.id)]
created = False
else:
ms, created = Membership.objects.get_or_create(member=member, party=party)
memberships[(member.id, party.id)] = ms
# if created: # again, unicode magic
# ms = Membership.objects.get(member=m,party=p)
# again, update the dates on the membership
if (ms.start_date is None) or (ms.start_date > vote_date):
ms.start_date = vote_date
if (ms.end_date is None) or (ms.end_date < vote_date):
ms.end_date = vote_date
if created: # save on first time, so it would have an id, be able to link, etc. all other updates are saved in the end
ms.save()
# add the current member's vote
va, created = VoteAction.objects.get_or_create(vote=v, member=member, type=vote,
party=member.current_party)
if created:
va.save()
logger.debug("done")
logger.debug(
"saving data: %d parties, %d members, %d memberships " % (len(parties), len(members), len(memberships)))
for party in parties:
parties[party].save()
for member in members:
members[member].save()
for ms in memberships:
memberships[ms].save()
logger.debug("done")
f.close()
except Exception:
logger.exception('Update db from file exception')
def update_members_from_file(self):
logger.debug('update_members_from_file')
f = gzip.open(os.path.join(DATA_ROOT, 'members.tsv.gz'))
content = f.read().split('\n')
for line in content:
if len(line) <= 1:
continue
(member_id, name, img_url, phone, fax, website, email,
family_status, number_of_children, date_of_birth,
year_of_birth, place_of_birth, date_of_death,
year_of_aliyah, k18, k19, _) = line.split('\t')
if email != '':
email = email.split(':')[1]
try:
if date_of_birth.find(',') >= 0:
date_of_birth = date_of_birth.split(',')[1].strip(' ')
date_of_birth = datetime.datetime.strptime(date_of_birth, "%d/%m/%Y")
except:
date_of_birth = None
try:
if date_of_birth.find(',') >= 0:
date_of_death = date_of_birth.split(',')[1].strip(' ')
date_of_death = datetime.datetime.strptime(date_of_death, "%d/%m/%Y")
except:
date_of_death = None
try:
year_of_birth = int(year_of_birth)
except:
year_of_birth = None
try:
year_of_aliyah = int(year_of_aliyah)
except:
year_of_aliyah = None
try:
number_of_children = int(number_of_children)
except:
number_of_children = None
try:
m = Member.objects.get(id=member_id)
m.phone = phone
m.fax = fax
m.email = email
m.family_status = family_status
m.number_of_children = number_of_children
m.date_of_death = date_of_death
m.save()
logger.debug('updated member %d' % m.id)
except Member.DoesNotExist: # member_id not found. create new
m = Member(id=member_id, name=name, img_url=img_url, phone=phone, fax=fax, website=None, email=email,
family_status=family_status,
number_of_children=number_of_children, date_of_birth=date_of_birth,
place_of_birth=place_of_birth,
date_of_death=date_of_death, year_of_aliyah=year_of_aliyah)
m.save()
m = Member.objects.get(pk=member_id) # make sure we are are
# working on the db object. e.g m.id is a number.
logger.debug('created member %d' % m.id)
if len(website) > 0:
l = Link(title='אתר האינטרנט של %s' % name, url=website,
content_type=ContentType.objects.get_for_model(m), object_pk=str(m.id))
l.save()
if k19: # KNESSET 19 specific
parties = Party.objects.filter(knesset_id=19).values_list('name', 'id')
k19 = k19.decode(ENCODING)
for k, v in parties:
if k in k19:
m.current_party_id = int(v)
logger.debug('member %s, k19 %s, party %s'
% (m.name,
k19,
k))
m.save()
if m.current_party is None:
logger.debug('member %s, k19 %s not found' %
(m.name,
k19))
