def generate_event(self, event_num): e = Event() min_date = (date.today() + timedelta(days=-5)).toordinal() max_date = (date.today() + timedelta(days=5)).toordinal() e.start_date = random_day = date.fromordinal(random.randint(min_date, max_date)) e.end_date = e.start_date + timedelta(0, 60 * 60) # duration 60 minutes e.title = 'Event %d' % event_num e.descr = 'Description for event %d' % event_num e.lat = random.randint(24000, 26000)/1000.0 e.lon = random.randint(58000, 62000)/1000.0 ''' e.street_address = models.CharField(max_length=500) e.city = models.CharField(max_length=500) e.country = models.CharField(max_length=100) ''' e.org_name = 'Sample Organizer' e.org_email = '*****@*****.**' e.org_phone = '+358501234567' e.save() return
def update_application(self, request, deserialized):
user = request.user
application = Application.objects.get(pk=deserialized['appId'])
try:
if deserialized['confirm']:
if application.owner_id == user.account_id:
application.is_confirmed_by_owner = 'YES'
else:
application.is_confirmed_by_tenant = 'YES'
if application.is_confirmed_by_tenant == application.is_confirmed_by_owner == 'YES':
application.status = 'CONFIRMED'
except:
pass
application.ejari_no = deserialized['leaseApplicationDetails'][
'contractNo']
application.start_date = deserialized['leaseApplicationDetails'][
'contractStartDate']
application.end_date = deserialized['leaseApplicationDetails'][
'contractEndDate']
application.address = deserialized['leaseApplicationDetails'][
'address']
application.premis_no = deserialized['leaseApplicationDetails'][
'premiseNo']
application.total_contract_value = deserialized[
'leaseApplicationDetails']['securityDepositAmount']
application.save()
registrationEvent = Event(
referenceid=application.id,
what="APPLICATION CONFIRMATION",
who=user.account_id,
)
registrationEvent.save()
def test_event_creation(self):
user = User(
name='Johannes',
password='******',
last_login=timezone.now(),
email='*****@*****.**',
)
user.save()
agent = Agent(
name='CloudVM',
status=True,
env='01234567890123456789',
version='12345',
address='10.0.0.1',
)
agent.save()
event = Event(
level='none',
data='Some data test',
arquivado=False,
date=timezone.now(),
agent=agent,
user=user,
)
event.full_clean()
event.save()
ev = Event.objects.get(pk=event.pk)
self.assertEqual(event.level, ev.level)
self.assertEqual(event.agent.name, ev.agent.name)
self.assertEqual(event.user.name, ev.user.name)
def create_event(type, owners, object_id):
"""
Создаёт событие
"""
event = Event()
event.type = type
event.object_id = object_id
event.save()
for owner in owners:
event.owners.add(owner)
event.save()
return event
def run():
y = 0
while y < 100:
longitude = random.uniform(17.0, 18.0)
lat = random.uniform(60.0, 61.0)
A = GEOSGeometry('POINT('+str(longitude)+' ' + str(lat)+')', srid=4326)
e = Event(user=MyUser.objects.get(pk=1), title="Lunch", description="En riktigt god lunch utlovas, kyckling och potatis. Kommer endast att kosta runt 30 kr.",
numOfPeople=random.randint(3,12), date_start="2016-07-"+str(random.randint(1,18)) +" 13:00:00.00Z", price=3, location=A)
e.save()
print y
y += 1
def create_event(event_creator, category, active=True, id=5):
event = Event(id=id,
title="test title default",
description="test description default",
venue="test venue",
start_date=date,
end_date=date,
creator=event_creator,
social_event=category,
active=active,
created_at=date)
event.save()
return event
def handle(self, *args, **options):
url = options['url']
scraper.scraper(url, scraper_op)
for i in golfers:
# print first
try:
g = Golfer.objects.get(name=i)
except Golfer.DoesNotExist:
print i + " does not exist. Creating..."
g = Golfer(name=i)
g.save()
name = raw_input('Name of event?: ')
event = Event(name=name, active=True)
event.save()
def add_post_to_db(posts):
category_q = Category.objects.values('name')
categories = []
for item in category_q:
categories.append(item['name'])
for post in posts:
if post["name"] and post["image"] and post["category"] \
and post["date"] and post["address"] \
and post["price"] and post["description"] and post["link"]:
e = Event()
e.name = post["name"],
e.image = post["image"],
e.category = Category.objects.get(id=1),
e.date = post["date"],
e.address = post["address"],
e.price = post["price"],
e.description = post["description"],
e.link = post["link"]
e.save()
print(e)
def save_events(events):
for e in events:
# deal with some events not having a published date
if 'published' in e:
published = pytz.utc.localize(datetime.strptime(e['published'],'%Y-%m-%dT%H:%M:%SZ'))
else:
published = None
#why have a paid event that doesn't have a price??
min_price = 0.0
max_price = 0.0
if 'ticket_availability' in e and e['ticket_availability'] is not None:
if 'minimum_ticket_price' in e['ticket_availability'] and e['ticket_availability']['minimum_ticket_price'] is not None:
if 'major_value' in e['ticket_availability']['minimum_ticket_price']:
min_price = e['ticket_availability']['minimum_ticket_price']['major_value']
if 'maximum_ticket_price' in e['ticket_availability'] and e['ticket_availability']['maximum_ticket_price'] is not None:
if 'major_value' in e['ticket_availability']['maximum_ticket_price']:
max_price = e['ticket_availability']['maximum_ticket_price']['major_value']
event = Event(
id = e['id'],
name = e['name']['text'],
description = e['description']['text'],
url = e['url'],
start = datetime.strptime(e['start']['utc'],'%Y-%m-%dT%H:%M:%SZ').astimezone(gettz(e['start']['timezone'])),
end = datetime.strptime(e['end']['utc'],'%Y-%m-%dT%H:%M:%SZ').astimezone(gettz(e['end']['timezone'])),
org_id = e['organization_id'],
date_created = pytz.utc.localize(datetime.strptime(e['created'],'%Y-%m-%dT%H:%M:%SZ')),
date_modified = pytz.utc.localize(datetime.strptime(e['changed'],'%Y-%m-%dT%H:%M:%SZ')),
date_published = published,
status = e['status'],
currency = e['currency'],
online_event = e['online_event'],
hide_start_date = e['hide_start_date'],
hide_end_date = e['hide_end_date'],
min_price = min_price,
max_price = max_price,
)
with transaction.atomic():
event.save()
def handle_event_proxy(event, chain):
event_key = get_event_key(event)
existed_event = Event.query.filter(Event.key == event_key).first()
if existed_event:
return
serialized_event = dict(
event.__dict__,
**{
'args': event['args'].__dict__,
'transactionHash': event['transactionHash'].hex(),
'blockHash': event['blockHash'].hex(),
},
)
# TODO: event handler
print(chain, event['event'])
new_event = Event(key=event_key,
chain=chain,
type=event['event'],
content=json.dumps(serialized_event))
new_event.save()
def obj_create(self, bundle, request=None, **kwargs):
"""
Updates the status of the Application
"""
applicationID = bundle.data['applicationID']
userID = bundle.data['userID']
try:
application_to_confirm = Application.objects.get(
internal_id=applicationID) ##get application object
tenant_id = application_to_confirm.tenant_id
owner_id = application_to_confirm.owner_id
## Update the Confirmation Field
if userID == tenant_id:
application_to_confirm.is_confirmed_by_tenant = "YES"
elif userID == owner_id:
application_to_confirm.is_confirmed_by_owner = "YES"
## Update status of application if both have been confirmed
if application_to_confirm.is_confirmed_by_tenant == "YES" and application_to_confirm.is_confirmed_by_owner == "YES":
application_to_confirm.status = "CONFIRMED"
application_to_confirm.save()
##Log the event
registrationEvent = Event(
referenceid=applicationID,
what="APPLICATION CONFIRMATION",
who=userID,
)
registrationEvent.save()
except Application.DoesNotExist: ##if doesn't exist create a new object
pass
def obj_create(self, bundle, request=None, **kwargs):
registrationForm = bundle.data['registrationForm']
tenantDetails = registrationForm['personalDetails']
ownerDetails = registrationForm['otherParty']
leaseApplicationDetails = registrationForm['leaseApplicationDetails']
depositDetails = registrationForm['depositDetails']
tenant_first_name = tenantDetails['firstName']
tenant_last_name = tenantDetails['lastName']
owner_first_name = ownerDetails['firstName']
owner_last_name = ownerDetails['lastName']
try: ##try to find if registrant user exist
tenant = User.objects.get(email=tenantDetails['email'])
generated_tenant_id = tenant.account_id
except User.DoesNotExist: ##if doesn't exist create a new object
##excuse my shit way of doing this, randomly generating user_id
random_uid = str(uuid.uuid4().hex)
generated_tenant_id = random_uid
#generated_tenant_id = str(tenant_first_name)[0] + str(tenant_last_name)[0] + random_uid[0] + random_uid[1] + random_uid[2] + random_uid[3]
generated_tenant_password = random_uid[4] + random_uid[
5] + random_uid[6] + random_uid[7] + str(
tenant_first_name)[0] + str(tenant_last_name)[0]
name = generate_random_username()
print('tenant username: '******'tenant password: '******'phoneNumber'],
email=tenantDetails['email'],
username=name)
tenant.set_password(generated_tenant_password)
tenant.save()
send_account_creation_email(tenant, generated_tenant_password)
##Log the event
registrationEvent = Event(
referenceid=generated_tenant_id,
what="TENANT REGISTRATION",
who=generated_tenant_id,
#when=bundle.data['createdAt']
)
registrationEvent.save()
try: ##try to find if owner user exist
owner = User.objects.get(email=ownerDetails['email'])
generated_owner_id = owner.account_id
except User.DoesNotExist: ##if doesn't exist create a new object
random_uid = str(uuid.uuid4().hex)
generated_owner_id = random_uid
#generated_owner_id = str(owner_first_name)[0] + str(owner_last_name)[0] + random_uid[0] + random_uid[1] + random_uid[2] + random_uid[3]
generated_owner_password = random_uid[4] + random_uid[
5] + random_uid[6] + random_uid[7] + str(
owner_first_name)[0] + str(owner_last_name)[0]
name = generate_random_username()
print('owner username: '******'owner password: '******'firstName'],
last_name=ownerDetails['lastName'],
contact_number=ownerDetails['phoneNumber'],
email=ownerDetails['email'],
username=name)
owner.set_password(generated_owner_password)
owner.save()
send_account_creation_email(owner, generated_owner_password)
##Log the event
registrationEvent = Event(
referenceid=generated_owner_id,
what="OWNER REGISTRATION",
who=generated_owner_id,
)
registrationEvent.save()
try: ##try to find if application allready exist
owner = Application.objects.get(
ejari_no=leaseApplicationDetails['contractNo'])
except Application.DoesNotExist: ##if doesn't exist create a new object
random_uid = str(uuid.uuid4().hex)
#generated_application_id = str(owner_first_name)[0] + str(tenant_first_name)[0] + \
# random_uid[0] + random_uid[1] + random_uid[2] + random_uid[3]
generated_application_id = random_uid
new_application = Application(
ejari_no=leaseApplicationDetails['contractNo'],
premis_no=leaseApplicationDetails['premiseNo'],
total_contract_value=leaseApplicationDetails[
'securityDepositAmount'],
address=leaseApplicationDetails['address'],
start_date=leaseApplicationDetails['contractStartDate'],
end_date=leaseApplicationDetails['contractEndDate'],
annual_rent=leaseApplicationDetails['annualRent'],
property_size=leaseApplicationDetails['propertySize'],
property_usage=leaseApplicationDetails['propertyUsage'],
currency_type=leaseApplicationDetails['currencyType'],
deposit_term=depositDetails['term'],
deposit_amount=depositDetails['amount'],
term_percent=depositDetails['termPercent'],
internal_id=generated_application_id,
tenant_id=generated_tenant_id,
owner_id=generated_owner_id)
new_application.save()
##Log the event
registrationEvent = Event(
referenceid=generated_application_id,
what="APPLICATION REGISTRATION",
who=generated_tenant_id,
)
registrationEvent.save()
send_application_confirm_email(tenant, owner, new_application)
def event_create(request):
if request.method == "POST":
uuid = request.POST.get("uuid", "")
longitude = request.POST.get("longitude", "")
latitude = request.POST.get("latitude", "")
scanner_name = request.POST.get("scanner_name", "")
try:
student = Student.objects.get(uuid=int(uuid))
except:
return JsonResponse({
"status": "error",
"message": "No student with that uuid",
"data": None
})
event_by_time = student.event_set.order_by('-time')
if len(event_by_time) != 0:
last_event = event_by_time[0]
time_elapsed = (timezone.now() - last_event.time).total_seconds()
if time_elapsed < 10:
return JsonResponse({
"status": "error",
"message": "An event was recently created for this user.",
"data": None
})
try:
event = Event(student=student,
longitude=float(longitude),
latitude=float(latitude),
scanner_name=scanner_name)
event.save()
# Send email if notifications turned on
if student.notifications:
parent_email = student.parent.email
subject_text = student.first_name + " checked into " + event.scanner_name + " just now"
content_text = "This is just a friendly notification - please use your account to see more detailed information. Thanks for using SafeStudent!"
sg = sendgrid.SendGridAPIClient(
apikey=os.environ.get('SENDGRID_API_KEY'))
from_email = Email("*****@*****.**")
to_email = Email(parent_email)
subject = subject_text
content = Content("text/plain", content_text)
mail = Mail(from_email, subject, to_email, content)
response = sg.client.mail.send.post(request_body=mail.get())
return JsonResponse({
"status": "success",
"message": None,
"data": event.id
})
except Exception as e:
return JsonResponse({
"status": "error",
"message": str(e),
"data": None
})
return JsonResponse({
"status": "error",
"message": "dude it needs to be a post request",
"data": None
})
def obtainTicketsFromGestionWebsite(self, ip):
url = self.heroku_url + 'tickets/'
headers = self.headers
headers['ipAddress'] = ip
try:
tickets = Ticket.objects.all()
auditoriums = Auditorium.objects.all()
events = Event.objects.all()
print('GET: ' + url)
response = requests.get(url, headers=self.headers, timeout=10)
f = open('log/GET_TICKET_LIST_GESTION.log', 'w')
f.write('GET: ' + url + ' | Headers: ' + str(headers) +
'\nResponse\n')
f.write(response.text)
f.close()
tickets_dict = response.json()
audi_do_not_exist = True
event_do_not_exist = True
if not isinstance(tickets_dict, str):
for ticket_dict in tickets_dict:
tickets = Ticket.objects.all()
auditoriums = Auditorium.objects.all()
events = Event.objects.all()
ticketHash = ticket_dict.get('id')
owner = ticket_dict.get('owner')
ticket_status = 'Non-Validated'
event_dict = ticket_dict.get('event')
event_name = event_dict.get('name')
event_time = event_dict.get('startDate')
auditorium_dict = event_dict.get('auditorium')
auditorium_name = auditorium_dict.get('name')
auditorium_address = auditorium_dict.get('address')
auditorium = Auditorium(name=auditorium_name,
address=auditorium_address)
for audi_in_db in auditoriums:
if audi_in_db.name == auditorium_name and audi_in_db.address == auditorium_address:
audi_do_not_exist = False
auditorium = audi_in_db
break
if audi_do_not_exist:
auditorium.save()
event = Event(name=event_name,
time=event_time,
auditorium=auditorium)
for event_in_db in events:
if event_in_db.name == event_name:
event_do_not_exist = False
event = event_in_db
break
if event_do_not_exist:
event.save()
ticket = Ticket(ticketHash=ticketHash,
status=ticket_status,
owner=owner,
event=event)
for tick in tickets:
if tick.ticketHash == ticketHash:
ticket = tick
ticket.save()
else:
print('Received: No event found retry in 1 sec')
time.sleep(1)
except requests.exceptions.Timeout:
print('TIMEOUT OBTAIN TICKET LIST')
except ValueError:
print('INVALID JSON FORMAT')
print(response.text)
def extract_events(self, url):
event_urls = []
events = []
handle = urllib2.urlopen(url)
dom = parse(handle)
sanitizer_re = re.compile(r'(style|id|class)="[^"]*"')
items = dom.getElementsByTagName('item')
for item in items:
e = Event()
e.remote_source_name = 'AaltoEvents'
e.remote_url = self.getdata(item, 'link')
if Event.objects.filter(remote_url__iexact=e.remote_url).count() > 0:
print 'Event already exists, continuing.'
continue
e.title = smart_unicode(self.getdata(item, 'title'))
e.descr = smart_unicode(self.getdata(item, 'description'))
e.descr = sanitizer_re.sub('', e.descr) # Strip style, id, class attrs
try:
e.start_date = dateutil.parser.parse(self.getdata(item, 'xcal:dtstart'))
except Exception:
print 'Err'
try:
e.end_date = dateutil.parser.parse(self.getdata(item, 'xcal:dtend'))
except Exception:
print 'Err'
try:
point = self.getdata(item, 'georss:point')
if not point: raise Exception('No georss:point')
point = point.replace(',', '.')
point = point.split(' ')
e.lat = float(point[0])
e.lon = float(point[1])
except Exception as ex:
#print '%s' % ex
e.lat = 0
e.lon = 0
e.org_name = smart_unicode(self.getdata(item, 'author'))
try:
#print e.lat, e.lon
if not (e.lat and e.lat != 0 and e.lon and e.lon != 0):
raise Exception('Missing lat or lon')
res = reverse_geocode(e.lat, e.lon)
#print res
if hasattr(res, 'street_address'):
e.street_address = res.street_address
else:
e.street_address = ''
if hasattr(res, 'city'):
e.city = res.city
else:
e.city = ''
if hasattr(res, 'country'):
e.country = res.country
else:
e.country = 'Finland'
except Exception as ex:
#print 'Error fetching street address: %s' % ex
e.street_address = ''
e.city = ''
e.country = ''
e.org_email = ''
e.org_phone = ''
e.save()
events.append(e)
return events
def obtainTicketList(self, ipAddress):
try:
tickets = Ticket.objects.all()
auditoriums = Auditorium.objects.all()
events = Event.objects.all()
url = 'https://' + ipAddress + '/api/tickets/'
print('GET: ' + url)
response = requests.get(url,
headers=self.headers,
timeout=2,
verify=False)
f = open('log/GET_TICKET_LIST_INTERNAL.log', 'w')
f.write('GET: ' + url + '\nResponse\n')
f.write(response.text)
f.close()
tickets_dict = response.json()
audi_do_not_exist = True
event_do_not_exist = True
if tickets_dict and 'NotReady' not in tickets_dict:
for ticket_dict in tickets_dict:
tickets = Ticket.objects.all()
auditoriums = Auditorium.objects.all()
events = Event.objects.all()
ticketHash = ticket_dict.get('ticketHash')
owner = ticket_dict.get('owner')
ticket_status = ticket_dict.get('status')
validationTime = ticket_dict.get('validationTime')
event_dict = ticket_dict.get('event')
event_name = event_dict.get('name')
event_time = event_dict.get('time')
auditorium_dict = event_dict.get('auditorium')
auditorium_name = auditorium_dict.get('name')
auditorium_address = auditorium_dict.get('address')
auditorium = Auditorium(name=auditorium_name,
address=auditorium_address)
for audi_in_db in auditoriums:
if audi_in_db.name == auditorium_name and audi_in_db.address == auditorium_address:
audi_do_not_exist = False
auditorium = audi_in_db
break
if audi_do_not_exist:
auditorium.save()
event = Event(name=event_name,
time=event_time,
auditorium=auditorium)
for event_in_db in events:
if event_in_db.name == event_name:
event_do_not_exist = False
event = event_in_db
break
if event_do_not_exist:
event.save()
ticket = Ticket(ticketHash=ticketHash,
status=ticket_status,
validationTime=validationTime,
owner=owner,
event=event)
for tick in tickets:
if tick.ticketHash == ticketHash:
ticket = tick
ticket.save()
else:
print('Received: No event found retry in 1 sec')
time.sleep(1)
except requests.exceptions.Timeout:
print('TIMEOUT INTERNAL OBTAIN TICKET LIST')
def populate_db(apps, schema_editor):
# set up some users
from django.contrib.auth.models import User
user1 = User.objects.create_user(username='******', email='*****@*****.**', password='******',
last_login=datetime.datetime.now())
user1.save()
user2 = User.objects.create_user(username='******', email='*****@*****.**', password='******',
last_login=datetime.datetime.now())
user2.save()
user3 = User.objects.create_user(username='******', email='*****@*****.**', password='******',
last_login=datetime.datetime.now())
user3.save()
user4 = User.objects.create_user(username='******', email='*****@*****.**', password='******',
last_login=datetime.datetime.now())
user4.save()
user5 = User.objects.create_user(username="******", email="*****@*****.**", password="******", last_login=datetime.datetime.now())
user5.save()
superuser = User.objects.create_superuser(username="******", email="*****@*****.**", password="******", last_login=datetime.datetime.now())
superuser.save()
# Now performed automatically by signals
# from api.models import Balance
# b1 = Balance(owner=user1, value=50)
# b1.save()
# b2 = Balance(owner=user2, value=50)
# b2.save()
# b3 = Balance(owner=user3, value=50)
# b3.save()
# b4 = Balance(owner=user4, value=50)
# b4.save()
# b5 = Balance(owner=superuser, value=250)
# b5.save()
# set up some events
from api.models import Event
e1 = Event(name='USU v. BYU', created=datetime.datetime.now(),
startTime=datetime.datetime(2021, 5, 17, 19),
endTime=datetime.datetime(2021, 5, 17, 22),
address='900 E 900 N, Logan, UT 84322'
)
e1.save()
e2 = Event(name='USU Wind Symphony', created=datetime.datetime.now(),
startTime=datetime.datetime(2021, 5, 20, 18),
endTime=datetime.datetime(2021, 5, 20, 20),
address='600 E 1150 N, North Logan, UT 84341'
)
e2.save()
# set up some lots
from api.models import ParentLot
p1 = ParentLot(owner=superuser,
name='Maverik Stadium',
address='875 Douglas Dr, Logan, UT 84321',
created=datetime.datetime.now(),
capSmallMax=80,
capMediumMax=40,
capLargeMax=10
)
p1.save()
p2 = ParentLot(owner=user4,
name='Gr8 Parking',
address='Engineering Lab Walkway, Logan, UT 84321',
created=datetime.datetime.now(),
capSmallMax=30,
capMediumMax=5,
capLargeMax=0
)
p2.save()
# assign some lots to events
from api.models import Lot
l1 = Lot(
owner=superuser,
openTime=datetime.time(17, 0, 0),
closeTime=datetime.time(23, 0, 0),
costSmall=10.00,
capSmallActual=50,
capSmallMax=50,
costMedium=15.00,
capMediumActual=20,
capMediumMax=20,
costLarge=20.00,
capLargeActual=10,
capLargeMax=10,
event=e1,
parentLot=p1
)
l1.save()
l2 = Lot(
owner=user4,
openTime=datetime.time(14, 0, 0),
closeTime=datetime.time(22, 0, 0),
costSmall=10.00,
capSmallActual=50,
capSmallMax=50,
costMedium=15.00,
capMediumActual=20,
capMediumMax=20,
costLarge=0,
capLargeActual=0,
capLargeMax=0,
event=e2,
parentLot=p2
)
l2.save()
# a couple reservations
from api.models import Reservation
r1 = Reservation(
owner=user1,
lot=l2,
size='small',
event=l2.event
)
r1.save()
r2 = Reservation(
owner=user3,
lot=l1,
size='medium',
event=l1.event
)
r2.save()
# create some groups
from django.contrib.auth.models import Group
owners = Group(id=1, name="Owners")
owners.save()
attendants = Group(id=2, name="Attendants")
attendants.save()
user4.groups.add(owners)
user5.groups.add(attendants)
timestamp = df[df.columns[6]][i]
date = timestamp.to_pydatetime()
stake = df[df.columns[1]][i]
first = df[df.columns[2]][i]
second = df[df.columns[3]][i]
third = df[df.columns[4]][i]
print(host)
if host:
p = Profile.objects.get(nickname=host[:-1])
try:
e = Event.objects.get(host=p, date=date)
except:
print("No event")
e = Event(league=league, host=p, date=date, status='F')
e.save()
g = None
try:
g = Game.objects.get(event=e, number=int(game))
except:
print("No game")
try:
g = Game(event=e, number=int(game), stake=stake)
g.save()
except:
pass
if g:
j = 0
for player in df.columns[7:-3]:
placing = df[df.columns[7 + j]][i]
j = j + 1
def handle(self, *args, **options):
# get latest
filename = get_dbfile()
# numeric details records
details_rc = extract_table(filename, 'EW_Report_NumericDetails')
# check for 1 record for each field report
fids = [r['ReportID'] for r in details_rc]
if len(set(fids)) != len(fids):
raise Exception(
'More than one NumericDetails record for a field report')
# numeric details records
details_gov = extract_table(filename, 'EW_Report_NumericDetails_GOV')
# check for 1 record for each field report
fids = [r['ReportID'] for r in details_gov]
if len(set(fids)) != len(fids):
raise Exception(
'More than one NumericDetails record for a field report')
# information
info_table = extract_table(filename, 'EW_Report_InformationManagement')
fids = [r['ReportID'] for r in info_table]
if len(set(fids)) != len(fids):
raise Exception(
'More than one InformationManagement record for a field report'
)
# ## many-to-many
# actions taken
actions_national = extract_table(filename,
'EW_Report_ActionTakenByRedCross')
actions_foreign = extract_table(filename,
'EW_Report_ActionTakenByPnsRC')
actions_federation = extract_table(
filename, 'EW_Report_ActionTakenByFederationRC')
# source types
source_types = extract_table(filename, 'EW_lofSources')
for s in source_types:
SourceType.objects.get_or_create(
pk=s['SourceID'], defaults={'name': s['SourceName']})
source_table = extract_table(filename, 'EW_Reports_Sources')
# disaster response
dr_table = extract_table(filename, 'EW_DisasterResponseTools')
# check for 1 record for each field report
fids = [r['ReportID'] for r in dr_table]
if len(set(fids)) != len(fids):
raise Exception(
'More than one DisasterResponseTools record for a field report'
)
# contacts
contacts = extract_table(filename, 'EW_Report_Contacts')
# field report
reports = extract_table(filename, 'EW_Reports')
rids = [r.rid for r in FieldReport.objects.all()]
num_reports_created = 0
logger.info('%s reports in database' % len(reports))
for i, report in enumerate(reports):
# Skip reports that we've already ingested.
# We don't have to update them because field reports can't be updated in DMIS.
rid = report['ReportID']
if rid in rids:
continue
report_name = report['Summary']
report_description = report['BriefSummary']
report_dtype = DisasterType.objects.get(
pk=PK_MAP[report['DisasterTypeID']])
record = {
'rid':
rid,
'summary':
report_name,
'description':
report_description,
'dtype':
report_dtype,
'status':
report['StatusID'],
'request_assistance':
report['GovRequestsInternAssistance'],
'actions_others':
report['ActionTakenByOthers'],
'report_date':
datetime.strptime(report['Inserted'],
REPORT_DATE_FORMAT).replace(tzinfo=pytz.utc),
}
details = fetch_relation(details_rc, report['ReportID'])
assert (len(details) <= 1)
if len(details) > 0:
details = details[0]
record.update({
'num_injured':
details['NumberOfInjured'],
'num_dead':
details['NumberOfCasualties'],
'num_missing':
details['NumberOfMissing'],
'num_affected':
details['NumberOfAffected'],
'num_displaced':
details['NumberOfDisplaced'],
'num_assisted':
details['NumberOfAssistedByRC'],
'num_localstaff':
details['NumberOfLocalStaffInvolved'],
'num_volunteers':
details['NumberOfVolunteersInvolved'],
'num_expats_delegates':
details['NumberOfExpatsDelegates']
})
details = fetch_relation(details_gov, report['ReportID'])
assert (len(details) <= 1)
if len(details) > 0:
details = details[0]
record.update({
'gov_num_injured':
details['NumberOfInjured_GOV'],
'gov_num_dead':
details['NumberOfDead_GOV'],
'gov_num_missing':
details['NumberOfMissing_GOV'],
'gov_num_affected':
details['NumberOfAffected_GOV'],
'gov_num_displaced':
details['NumberOfDisplaced_GOV'],
'gov_num_assisted':
details['NumberOfAssistedByGov_GOV']
})
info = fetch_relation(info_table, report['ReportID'])
if len(info) > 0:
info = {k: '' if v is None else v for k, v in info[0].items()}
record.update({
'bulletin': {
'': 0,
'None': 0,
'Planned': 2,
'Published': 3
}[info['InformationBulletin']],
'dref': {
'': 0,
'No': 0,
'Planned': 2,
'Yes': 3
}[info['DREFRequested']],
'dref_amount':
0 if info['DREFRequestedAmount'] == '' else float(
info['DREFRequestedAmount']),
'appeal': {
'': 0,
'Planned': 2,
'Yes': 3,
'NB': 0,
'No': 0,
'YES': 3
}[info['EmergencyAppeal']],
'appeal_amount':
0 if info['EmergencyAppealAmount'] == '' else float(
info['EmergencyAppealAmount']),
})
# disaster response
response = fetch_relation(dr_table, report['ReportID'])
if len(response) > 0:
response = {
k: '' if v is None else v
for k, v in response[0].items()
}
record.update({
'rdrt': {
'': 0,
'No': 0,
'Yes': 3,
'Planned/Requested': 2
}[response['RDRT']],
'fact': {
'': 0,
'No': 0,
'Yes': 3,
'Planned/Requested': 2
}[response['FACT']],
'eru_relief': {
'': 0,
'Yes': 3,
'Planned/Requested': 2,
'No': 0
}[response['ERU']]
})
field_report = FieldReport(**record)
# Create an associated event object
event_record = {
'name':
report_name if len(report_name) else report_dtype.name,
'summary':
report_description,
'dtype':
report_dtype,
'disaster_start_date':
datetime.utcnow().replace(tzinfo=timezone.utc),
'auto_generated':
True,
'auto_generated_source':
SOURCES['report_ingest'],
}
event = Event(**event_record)
event.save()
field_report.event = event
field_report.save()
num_reports_created = num_reports_created + 1
try:
country = Country.objects.select_related().get(
pk=report['CountryID'])
except ObjectDoesNotExist:
logger.warning('Could not find a matching country for %s' %
report['CountryID'])
country = None
if country is not None:
field_report.countries.add(country)
event.countries.add(country)
if country.region is not None:
# No need to add a field report region, as that happens through a trigger.
field_report.regions.add(country.region)
event.regions.add(country.region)
# ## add items with foreignkeys to report
# national red cross actions
actions = fetch_relation(actions_national, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='NTLS',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# foreign red cross actions
actions = fetch_relation(actions_foreign, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='PNS',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# federation red cross actions
actions = fetch_relation(actions_federation, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='FDRN',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# sources
sources = fetch_relation(source_table, report['ReportID'])
for s in sources:
spec = '' if s['Specification'] is None else s['Specification']
Source.objects.create(
stype=SourceType.objects.get(pk=s['SourceID']),
spec=spec,
field_report=field_report)
# disaster response
response = fetch_relation(dr_table, report['ReportID'])
# contacts
contact = fetch_relation(contacts, report['ReportID'])
if len(contact) > 0:
# make sure just one contacts record
assert (len(contact) == 1)
contact = contact[0]
fields = [
'Originator', 'Primary', 'Federation', 'NationalSociety',
'MediaNationalSociety', 'Media'
]
for f in fields:
if contact_is_valid(contact, f):
FieldReportContact.objects.create(
ctype=f,
name=contact['%sName' % f],
title=contact['%sFunction' % f],
email=contact['%sContact' % f],
field_report=field_report,
)
total_reports = FieldReport.objects.all()
logger.info('%s reports created' % num_reports_created)
logger.info('%s reports in database' % total_reports.count())
# org type mapping
org_types = {
'1': 'NTLS',
'2': 'DLGN',
'3': 'SCRT',
'4': 'ICRC',
}
last_login_threshold = timezone.now() - timedelta(days=365)
# add users
user_records = extract_table(filename, 'DMISUsers')
processed_users = 0
for i, user_data in enumerate(user_records):
if user_data['LoginLastSuccess'] == '':
continue
last_login = datetime.strptime(
user_data['LoginLastSuccess'],
REPORT_DATE_FORMAT,
)
last_login = pytz.UTC.localize(last_login)
# skip users who haven't logged in for a year
if last_login < last_login_threshold:
continue
try:
user = User.objects.get(username=user_data['UserName'])
except ObjectDoesNotExist:
user = None
if user is None:
name = user_data['RealName'].split()
first_name = name[0]
last_name = ' '.join(name[1:]) if len(name) > 1 else ''
user = User.objects.create(
username=user_data['UserName'],
first_name=first_name if len(first_name) <= 30 else '',
last_name=last_name if len(last_name) <= 30 else '',
email=user_data['EmailAddress'],
last_login=last_login,
)
user.set_password(user_data['Password'])
user.is_staff = True if user_data[
'UserIsSysAdm'] == '1' else False
# set user profile info
user.profile.org = user_data['OrgTypeSpec'] if len(
user_data['OrgTypeSpec']) <= 100 else ''
user.profile.org_type = org_types.get(user_data['OrgTypeID'])
# print(i, user_data['CountryID']) # - for debug purposes. Check go-api/data/Countries.csv for details.
if user_data['CountryID'] in ['275', '281']:
user_data[
'CountryID'] = '47' # Hong Kong or Taiwan should be handled as China. Macao (279) is other case.
elif user_data['CountryID'] in ['284']:
user_data[
'CountryID'] = '292' # Zone Central and West Africa -> Central Africa Country Cluster
user.profile.country = Country.objects.get(
pk=user_data['CountryID'])
user.profile.city = user_data['City'] if len(
user_data['City']) <= 100 else ''
user.profile.department = user_data['Department'] if len(
user_data['Department']) <= 100 else ''
user.profile.position = user_data['Position'] if len(
user_data['Position']) <= 100 else ''
user.profile.phone_number = user_data['PhoneNumberProf'] if len(
user_data['PhoneNumberProf']) <= 100 else ''
user.save()
processed_users = processed_users + 1
logger.info('%s updated active user records' % processed_users)
def handle(self, *args, **options):
print "Scraping AYY event calendar"
datere = re.compile(r"(\d+\.\d+\.\d+), (\d+)\.(\d+)")
baseurl = "http://ayy.fi/kalenteri/"
url = baseurl
fetched_main_pages = 0
while fetched_main_pages < 6 and url:
print "Getting month [current+%d]: %s" % (fetched_main_pages, url)
event_listing = urllib2.urlopen(url)
fetched_main_pages = fetched_main_pages + 1
soup = BeautifulSoup(event_listing.read())
try:
url = baseurl + soup.find(id="content").find_all("a")[1]["href"]
print "Next url: %s" % url
except Exception:
print "No next url found, this will be the last page"
url = None
for row in soup.find_all("tr"):
try:
link = row.a
if not link:
continue # No links found
detail_url = link["href"]
if detail_url == "":
continue # No valid url
print "\tParsing: %s" % (detail_url)
# Get data, parse
detail_data = urllib2.urlopen(detail_url)
dsoup = BeautifulSoup(detail_data.read())
event = Event()
event.remote_source_name = "ayy"
event.remote_url = detail_url
container = dsoup.find(id="content")
if not container:
continue
# Extract title
try:
event.title = container.find("h1", {"class": "main-title"}).string
except Exception as e:
print "\tError extracting title: %s" % e
# print "Title: %s" % (title)
# Extract description
try:
content = container.find("div", {"class": "entry-content"})
for c in content.find_all("script"):
c.extract() # Remove scripts
for c in content.find_all("div", {"class": "pd-rating"}):
c.extract()
event.descr = ""
for c in content.contents:
event.descr += str(c)
except Exception as e:
print "\tError extracting description: %s" % e
# print "Descr: %s" % (contentstr[:100])
# Extract times
try:
metadata = content.next_sibling.next_sibling.next_sibling
start_time = metadata.p
for c in start_time.find_all("b"):
c.extract() # Remove b tags == titles
start_time = start_time.get_text().split("\n") # Split remaining by line
end_time = start_time[1]
start_time = start_time[0]
# Check if dates contain . as time separator, replace with : if so
s = datere.match(start_time)
if s:
start_time = s.expand(r"\1 \2:\3")
s = datere.match(end_time)
if s:
end_time = s.expand(r"\1 \2:\3")
# Parse
event.start_date = parser.parse(start_time, dayfirst=True)
event.end_date = parser.parse(end_time, dayfirst=True)
# print "Start: %s, end: %s" % (event.start_date, event.end_date)
except Exception as e:
print "\tError resolving date: %s" % e
raise e # Fatal, dates are required
# Extract location
try:
info = metadata.contents[7].get_text().split("\n")
event.venue = info[0].split("Paikka: ")[1]
event.street_address = info[1].split("Osoite: ")[1]
event.city = info[2].split("Kaupunki: ")[1]
event.country = "Finland"
query = ""
if event.street_address:
query += "%s, " % self.normalize_street_address(event.street_address)
if event.city:
query += "%s, " % event.city
if event.country:
query += "%s" % event.country
query = smart_unicode(query)
geores = geocoder.geocode(query)
if geores:
event.lat = geores["lat"]
event.lon = geores["lon"]
else:
print "\tUnable to resolve coordinates for query %s" % query
except Exception as e:
print "\tError resolving location: %s" % e
# print 'Loc: %s, addr: %s, city: %s' % (loc, addr, city)
# Extract links
try:
metadata = metadata.next_sibling.next_sibling
info = metadata.contents[3]
links = info.find_all("a")
homepage = links[0]["href"]
facebookev = links[1]["href"]
except Exception as e:
print "\tError resolving links: %s" % e
# print 'Homepage: %s, FB: %s' % (homepage, facebookev)
# Extract contact info
try:
info = metadata.contents[7].get_text().split("\n")
event.org_name = info[0].split(u"Järjestäjä: ")[1]
event.org_email = info[1].split(u"Sähköposti: ")[1]
event.org_phone = info[2].split("Puhelin: ")[1]
except Exception as e:
print "\tError resolving organizer info: %s" % e
# print 'Name: %s, email: %s, phone: %s' % (org_name, org_email, org_phone)
event.save()
except Exception as e:
print "\tFATAL ERROR handling event, discarded"
# print '----------------------------------------'
# break
return
