def _findItem(session, url, caldata):
now = datetime.now()
startdate = (now - timedelta(days=7)).strftime(DATEFORMAT)
enddate = (now + timedelta(days=30)).strftime(DATEFORMAT)
folderid = utils.rget(caldata, 'Body.CalendarFolder.FolderId.Id', None)
headers = {
'action':
'FindItem',
'x-owa-urlpostdata':
'%7B%22__type%22%3A%22FindItemJsonRequest%3A%23Exchange%22%2C%22Header%22%3A%7B%22__type%22%3A%22'
'JsonRequestHeaders%3A%23Exchange%22%2C%22RequestServerVersion%22%3A%22Exchange2015%22%2C%22TimeZoneContext%22%3A%7B%22' # noqa
'__type%22%3A%22TimeZoneContext%3A%23Exchange%22%2C%22TimeZoneDefinition%22%3A%7B%22__type%22%3A%22'
'TimeZoneDefinitionType%3A%23Exchange%22%2C%22Id%22%3A%22Pacific%20Standard%20Time%22%7D%7D%7D%2C%22Body%22%3A%7B%22'
'__type%22%3A%22FindItemRequest%3A%23Exchange%22%2C%22ParentFolderIds%22%3A%5B%7B%22__type%22%3A%22FolderId%3A%23'
f'Exchange%22%2C%22Id%22%3A%22{folderid}%22%7D%5D%2C%22'
'ItemShape%22%3A%7B%22__type%22%3A%22ItemResponseShape%3A%23Exchange%22%2C%22BaseShape%22%3A%22IdOnly%22%7D%2C%22Traversal%22%3A%22'
'Shallow%22%2C%22Paging%22%3A%7B%22__type%22%3A%22CalendarPageView%3A%23Exchange%22%2C%22'
f'StartDate%22%3A%22{startdate}%22%2C%22EndDate%22%3A%22{enddate}%22%7D%7D%7D',
}
url = url.replace('/calendar/published/', '/owa/calendar/')
url = url.replace('/calendar.html', '/service.svc')
log.info('Fetching events from %s', url)
response = session.post(url, headers=headers).json()
return utils.rget(response,
'Body.ResponseMessages.Items.0.RootFolder.Items', [])
def disconnect(self, provider):
""" Disconnect the specified provider. """
if provider.lower() == 'google':
self.google_email = ''
self.google_creds = ''
self.save()
log.info('Disconnected Google for %s', self.email)
def auth_google(request, google_code):
""" Authenticate with Google and return the user.
https://developers.google.com/identity/sign-in/web/server-side-flow
https://developers.google.com/gmail/api/auth/web-server
"""
creds = client.credentials_from_code(settings.GOOGLE_CLIENTID,
settings.GOOGLE_SECRET,
settings.GOOGLE_SCOPES,
google_code)
creds.authorize(httplib2.Http())
google_email = creds.id_token['email']
# Check user is currently logged in
if request.user and request.user.is_active:
request.user.google_email = creds.id_token['email']
request.user.google_creds = creds.to_json()
request.user.save()
log.info('Connected Google account %s to userid %s', google_email,
request.user)
return request.user
# User is not currently logged in
user = User.objects.get(google_email=google_email)
if user and user.is_active:
request.user.google_creds = creds.to_json()
login(request, user)
log.info('Logged in via Google as %s', google_email)
return user
def auth_django(request, email, password):
""" Authenticate with Django and return the user. """
username = utils.get_object_or_none(User, email=email).username
user = authenticate(username=username, password=password)
if user and user.is_active:
login(request, user)
log.info('Logged in via Django as %s', user.email)
return user
def google_auth(self):
if self.google_creds:
creds = client.OAuth2Credentials.from_json(self.google_creds)
if creds and creds.access_token_expired:
log.info('Refreshing Google creds for %s', self.google_email)
creds.refresh(httplib2.Http())
httpauth = creds.authorize(httplib2.Http())
return creds, httpauth
raise Exception('Not logged into Google.')
def _get_page(self, userid, page):
photos = []
log.info('Fetching 500px photos: %s page %s' % (userid, page))
url = self.FEED.format(userid=userid, page=page, rpp=self.RPP)
response = self.session.get(url)
for photo in response.json()['photos']:
if _filter(photo, 'width', 'height'):
photos.append(
_photo(photo, 'images.0.url', 'user.fullname', 'name',
'description'))
return photos
def _get_page(self, groupid, page):
photos = []
log.info('Fetching Flickr photos: %s page %s' % (groupid, page))
response = self.flickr.groups.pools.getPhotos(extras=self.EXTRAS,
get_user_info=1,
group_id=groupid,
page=page,
per_page=self.RPP)
for photo in response.json()['photos']['photo']:
if _filter(photo, 'width_h', 'height_h'):
photos.append(
_photo(photo, 'url_h', 'ownername', 'title',
'description._content'))
return photos
def get_photos(self):
photos = []
for groupid in self.GROUPIDS:
response = self.flickr.groups.pools.getPhotos(group_id=groupid,
per_page=self.RPP)
pages = response.json()['photos']['pages']
kwargs = {
str(p): [self._get_page, groupid, p]
for p in range(1, pages + 1)
}
results = threaded(numthreads=20, **kwargs)
for page, result in results.items():
photos += result
log.info('Finished processing %s photos from Flickr' % len(photos))
return photos
def get_photos(self, rpp=RPP):
photos = []
for userid in self.USERIDS:
url = self.FEED.format(userid=userid, page=1, rpp=rpp)
response = self.session.get(url)
pages = response.json()['total_pages']
kwargs = {
str(p): [self._get_page, userid, p]
for p in range(1, pages + 1)
}
results = threaded(numthreads=10, **kwargs)
for page, result in results.items():
photos += result
log.info('Finished processing %s photos from 500px' % len(photos))
return photos
def _getAnonymousCalendarSessionData(session, url):
""" Returns the sessiojn data needed to render this calendar. """
headers = {
'action':
'GetAnonymousCalendarSessionData',
'x-owa-urlpostdata':
'%7B%22__type%22%3A%22GetAnonymousCalendarSessionDataJsonRequest%3A%23'
'Exchange%22%2C%22Header%22%3A%7B%22__type%22%3A%22JsonRequestHeaders%3A%23Exchange%22%2C%22' # noqa
'RequestServerVersion%22%3A%22Exchange2015%22%2C%22TimeZoneContext%22%3A%7B%22__type%22%3A%22'
'TimeZoneContext%3A%23Exchange%22%2C%22TimeZoneDefinition%22%3A%7B%22__type%22%3A%22TimeZoneDefinitionType%3A%23'
'Exchange%22%2C%22Id%22%3A%22Pacific%20Standard%20Time%22%7D%7D%7D%7D',
}
url = url.replace('/calendar/published/', '/owa/calendar/')
url = url.replace('/calendar.html', '/service.svc')
log.info('Fetching caldata from %s', url)
return session.post(url, headers=headers).json()
def import_qfx(self, user, filename, handle):
""" Import transactions from a qfx file. """
try:
self.files += 1
transactions = []
log.info('Importing transactions qfx file: %s' % filename)
qfx = OfxParser.parse(handle)
fid = int(qfx.account.institution.fid)
if fid not in self._accounts:
raise Exception('Not tracking account fid: %s' % fid)
account = self._accounts[fid]
# Update transactions
for trx in qfx.account.statement.transactions:
trx = trx.__dict__
trx['trxid'] = trx['id']
trx['accountfid'] = fid
if not self._transaction_exists(trx, addit=True):
transactions.append(
Transaction(
user=user,
account_id=account.id,
trxid=trx['id'],
payee=trx[account.payee or 'payee'],
amount=trx['amount'],
date=trx['date'].date(),
# original values
original_date=trx['date'].date(),
original_payee=trx[account.payee or 'payee'],
original_amount=trx['amount'],
))
self.label_transactions(transactions)
self.categorize_transactions(transactions)
log.info('Saving %s new transactions from qfx file: %s' %
(len(transactions), filename))
Transaction.objects.bulk_create(transactions)
self.status.append('%s: added %s transactions' %
(filename, len(transactions)))
self.transactions += len(transactions)
# Update account balance
statementdt = timezone.make_aware(qfx.account.statement.end_date)
if account.balancedt is None or statementdt > account.balancedt:
account.balance = qfx.account.statement.balance
account.balancedt = statementdt
account.save()
except Exception as err:
log.exception(err)
self.status.append('Error %s: %s' % (filename, err))
def save(self, *args, **kwargs):
# Reorder the categories if needed
if self.sortindex is None:
categories = Category.objects.order_by('-sortindex')
self.sortindex = categories[0].sortindex + 1 if categories.exists(
) else 0
elif self.sortindex != self._init_sortindex:
index = 0
log.info('Moving category %s to index %s', self.name,
self.sortindex)
categories = Category.objects.exclude(
id=self.id).order_by('sortindex')
self.sortindex = max(0, self.sortindex)
self.sortindex = min(self.sortindex, len(categories))
for catid in categories.values_list('id', flat=True):
index += 1 if index == self.sortindex else 0
Category.objects.filter(id=catid).update(sortindex=index)
index += 1
super(Category, self).save(*args, **kwargs)
def _check_send_email(self, record):
""" Return True if count is 0, we reached max_count, or timeout reached.
Otherwise just increment the count.
"""
now = time.time()
etype, evalue, tb = record.exc_info
if etype in self.ignore:
log.info(f'Not sending excpetion email; ignored')
return False
hashkey = f'exc_{self._get_ehash(tb)}'
count, timeout = self._get_cached(hashkey)
newcount = count + 1
if count == 0 or newcount >= self.maxcount or now >= timeout:
ename = etype.__name__
estr = str(evalue) or '--'
record.getMessage = lambda: f'[{newcount}x] {ename}: {estr}'
self._set_cached(hashkey, 1, now + self.timeout)
if settings.DEBUG:
log.info('Not sending excpetion email; DEBUG=True')
return False
return True
else:
log.info(f'Not sending excpetion email; count={newcount}')
self._set_cached(hashkey, newcount, timeout)
return False
def login(request, *args, **kwargs):
""" Allows logging in with various authentication schemes.
You must call this resource using a POST request.
* email/password - Login with a regular Django account.
* google_code - Login via Google (email must match django account).
"""
try:
email = request.data.get('email')
password = request.data.get('password')
google_code = request.data.get('google_code')
if email and password:
user = User.auth_django(request, email, password)
elif google_code:
user = User.auth_google(request, google_code)
if user and user.is_active:
serializer = AccountSerializer(user, context={'request': request})
return Response(serializer.data)
log.info('Unknown email or password: %s', email)
except Exception as err:
log.error(err, exc_info=1)
return Response({'status': 'Unknown email or password.'},
status=status.HTTP_403_FORBIDDEN)
def __enter__(self):
if self.label:
log.info(color('-' * 25, 'blue'))
log.info(color('%s - start of profiling' % self.label, 'blue'))
log.info(color('-' * 25, 'blue'))
self.sqltime, self.longest, self.numshown = 0.0, 0.0, 0
self.initqueries = len(connection.queries)
self.starttime = time.time()
return self
def post_save(sender, instance, created, *args, **kwargs):
if created:
log.info(f'Calling Django command: updatestocks --ticker={instance.ticker}')
call_command('updatestocks', ticker=instance.ticker)
def __exit__(self, *exc):
for query in connection.queries[self.initqueries:]:
self.sqltime += float(query['time'].strip('[]s'))
self.longest = max(self.longest, float(query['time'].strip('[]s')))
if self.show_queries:
if not self.filter or self.filter in query['sql']:
self.numshown += 1
querystr = color('[%ss] ' % query['time'], 'yellow')
querystr += color(query['sql'], 'blue')
log.info('')
log.info(querystr)
numqueries = len(connection.queries) - self.initqueries
numhidden = numqueries - self.numshown
runtime = round(time.time() - self.starttime, 3)
proctime = round(runtime - self.sqltime, 3)
log.info(color('-' * 8, 'blue'))
if self.label:
log.info(color('%s - end of profiling' % self.label, 'blue'))
log.info(color('-' * 8, 'blue'))
log.info(color('Total Time: %ss' % runtime, 'yellow'))
log.info(color('Proc Time: %ss' % proctime, 'yellow'))
log.info(
color(
'Query Time: %ss (longest: %ss)' %
(self.sqltime, self.longest), 'yellow'))
log.info(
color('Num Queries: %s (%s hidden)\n' % (numqueries, numhidden),
'yellow'))
