def run(self):
try:
access_token = UserSocialAuth.objects.get(user=self.user, provider='facebook').extra_data['access_token']
self.graph = get_facebook_graph(access_token=access_token)
if self.graph:
for n in range(0, 5):
try:
url = None
try:
if u'class' in self.params:
url = self.params[u'class']
elif u'link' in self.params:
url = self.params[u'link']
if url:
self.graph.set(u'/', {'ids' : url, 'scrape' : 'true'})
except OAuthException:
LoggingHelper.getDebugLogger().debug(u'Error while scraping {0}'.format(url), exc_info=1)
return self.graph.set(u'{0}/{1}'.format(self.facebook_id, self.action), self.params)
except PermissionException:
LoggingHelper.getDebugLogger().debug(u'Permission not granted (facebook_id : {0}, action {1}, params {2})'.format(self.facebook_id, self.action, self.params), exc_info=1)
except OAuthException:
LoggingHelper.getDebugLogger().debug(u'Refreshing token (facebook_id : {0}, action {1}, params {2})'.format(self.facebook_id, self.action, self.params), exc_info=1)
self.refresh_token()
except OpenFacebookException:
LoggingHelper.getErrorLogger().error(u'Error while posting on Facebook (facebook_id : {0}, action {1}, params {2})'.format(self.facebook_id, self.action, self.params), exc_info=1)
time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
except UserSocialAuth.DoesNotExist:
LoggingHelper.getDebugLogger().debug('No access token')
def facebook_task(user, facebook_id, action, params):
try:
user_social = UserSocialAuth.objects.get(user=user,
provider='facebook')
access_token = user_social.extra_data['access_token']
graph = get_facebook_graph(access_token=access_token)
if graph:
for n in range(0, 5):
try:
url = None
try:
if u'class' in params:
url = params[u'class']
elif u'link' in params:
url = params[u'link']
if url:
graph.set(u'/', {'ids': url, 'scrape': 'true'})
except OAuthException:
LoggingHelper.getDebugLogger().debug(
u'Error while scraping {0}'.format(url),
exc_info=1)
return graph.set(u'{0}/{1}'.format(facebook_id, action),
params)
except PermissionException:
LoggingHelper.getDebugLogger().debug(
u'Permission not granted (facebook_id : {0}, action {1}, params {2})'
.format(facebook_id, action, params),
exc_info=1)
except OAuthException:
LoggingHelper.getDebugLogger().debug(
u'Refreshing token (facebook_id : {0}, action {1}, params {2})'
.format(facebook_id, action, params),
exc_info=1)
try:
user_social.refresh_token()
except Exception:
pass
except OpenFacebookException:
LoggingHelper.getErrorLogger().error(
u'Error while posting on Facebook (facebook_id : {0}, action {1}, params {2})'
.format(facebook_id, action, params),
exc_info=1)
time.sleep((2**n) + (random.randint(0, 1000) / 1000))
if action in [
'{0}:invite_participate'.format(
settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME),
'{0}:invite_cowrite'.format(
settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME)
]:
facebook_task.apply_async([
user, facebook_id, '{0}:{1}'.format(
settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME,
'cowrite'), params
])
break
except UserSocialAuth.DoesNotExist:
LoggingHelper.getDebugLogger().debug('No access token')
def activation_emails_task():
logger.debug('Starting activation emails task')
users = User.objects.all().select_related('profile')
for user in users:
try:
if user.date_joined and user.last_login:
users_has_documents = Training.objects.filter(
creator_id=user).exists()
context = Context({'user': user})
today_at_midnight = timezone.now().replace(hour=0,
minute=0,
second=0)
logger.debug(user.date_joined)
if timezone.is_aware(user.date_joined):
date_joined = user.date_joined
else:
date_joined = timezone.make_aware(
user.date_joined, timezone.get_current_timezone())
days_since_signup = (today_at_midnight - date_joined).days
if days_since_signup == 2 and not users_has_documents:
logger.debug('Sending activation email to {0}'.format(
user.email))
context.update({
'ga_campaign_params':
'utm_source=unishared&utm_content=v1&utm_medium=e-mail&utm_campaign=activation_mail'
})
email_task.apply_async([
u'Create your first collaborative document', context,
'activation_mail', [user.email]
])
if user.get_profile().last_seen:
days_since_login = (today_at_midnight -
user.get_profile().last_seen).days
if days_since_login == 14:
logger.debug(
'Sending dropping out email to {0}'.format(
user.email))
context.update({
'ga_campaign_params':
'utm_source=unishared&utm_content=v1&utm_medium=e-mail&utm_campaign=dropping_out_mail'
})
email_task.apply_async([
u'Dropping out?', context, 'dropping_out_mail',
[user.email]
])
except:
LoggingHelper.getErrorLogger().error(
'An error occurred while sending activation email to user {0}'.
format(user.id),
exc_info=1)
def live_detection_failure(sender, **k):
LoggingHelper.getErrorLogger().error(
'An error occurred in task {0} with args {1}'.format(
sender, sender.request.args),
exc_info=1)
if sender is live_detection_change_feed:
with sender.app.connection() as conn:
sender.app.send_task(
'UniShared_python.website.tasks.live_detection_change_feed',
sender.request.args,
connection=conn)
def update_informations_task():
logger.debug('Starting update informations')
client = GoogleDriveHelper.get_docs_client()
params = {}
params['includeDeleted'] = False
result = client.changes().list(**params).execute()
total_updated = 0
for item in result['items']:
try:
resource_id = GoogleDriveHelper.get_id_from_feed_url(item.id.text)
if resource_id:
is_updated = False
title = item['file']['title']
updated = parser.parse(item['file']['modifiedDate'])
updated = calendar.timegm(updated.utctimetuple())
logger.debug('Working on resource {0}'.format(resource_id))
document = Training.objects.get(resource_id=resource_id)
if document.last_updated < updated:
logger.debug(
'Updating last_updated for resource {0} from {1} to {2}'
.format(resource_id, document.last_updated, updated))
document.last_updated = updated
is_updated = True
if document.title != title:
logger.debug(
u'Updating title for resource {0} from {1} to {2}'.
format(resource_id, document.title, title))
document.title = title
is_updated = True
if is_updated:
total_updated += 1
document.save()
except Training.DoesNotExist:
logger.debug('Unexisting resource {0}'.format(resource_id))
except Exception:
LoggingHelper.getErrorLogger().error(
'An error occured while updating {0}'.format(item.id.text),
exc_info=1)
logger.debug('{0} resources updated'.format(total_updated))
def run(self):
logger = LoggingHelper.getDebugLogger()
logger.debug('Looking for a temporary share for training {0} and user {1}'.format(self.training.resource_id, self.user))
trainingShare = TrainingTempShare.objects.filter(Q(facebook_id=self.user.get_profile().facebook_id) | Q(email=self.user.email), training=self.training)
if trainingShare.exists():
logger.debug('Temporary share found for training {0} and user {1}'.format(self.training.resource_id, self.user))
if self.training.creator != self.user and self.user not in self.training.cowriters.all():
logger.debug('Adding co-writer for training {0} and user {1}'.format(self.training.resource_id, self.user))
self.training.cowriters.add(self.user)
if self.user in self.training.participants.all():
logger.debug('Remove participant for training {0} and user {1}'.format(self.training.resource_id, self.user))
self.training.participants.remove(self.user)
trainingShare.delete()
elif self.training.creator != self.user and self.user not in self.training.cowriters.all() and self.user not in self.training.participants.all():
logger.debug('Adding participant for training {0} and user {1}'.format(self.training.resource_id, self.user))
self.training.participants.add(self.user)
if Training.objects.filter(participants=self.user).count() == 1:
logger.debug('Sending first participation mail')
context = Context({'username': self.user.username, 'first_name' : self.user.first_name, 'profile_url' : ProfileHelper.get_profile_url(self.user.username), 'document_url' : GoogleDriveHelper.get_document_url(self.training.resource_id), 'document_title' : self.training.title, 'ga_campaign_params' : 'utm_source=unishared&utm_content=v1&utm_medium=e-mail&utm_campaign=new_document_participation_mail'})
email_task.apply_async(["Yihaa, you participated in a collaborative document!", context, "new_document_participation_mail", [self.user.email]])
try:
logger.debug('Adding participation for training {0} and user {1}'.format(self.training.resource_id, self.user))
participation = TrainingParticipation.objects.get(user=self.user, training=self.training)
participation.count += 1
participation.save()
except TrainingParticipation.DoesNotExist:
TrainingParticipation.objects.create(user=self.user, training=self.training, count=1)
def update_informations_task():
logger.debug('Starting update informations')
client = GoogleDriveHelper.get_docs_client()
params = {}
params['includeDeleted'] = False
result = client.changes().list(**params).execute()
total_updated = 0
for item in result['items']:
try:
resource_id = GoogleDriveHelper.get_id_from_feed_url(item.id.text)
if resource_id:
is_updated = False
title = item['file']['title']
updated = parser.parse(item['file']['modifiedDate'])
updated = calendar.timegm(updated.utctimetuple())
logger.debug('Working on resource {0}'.format(resource_id))
document = Training.objects.get(resource_id=resource_id)
if document.last_updated < updated:
logger.debug(
'Updating last_updated for resource {0} from {1} to {2}'.format(resource_id, document.last_updated,
updated))
document.last_updated = updated
is_updated = True
if document.title != title:
logger.debug(
u'Updating title for resource {0} from {1} to {2}'.format(resource_id, document.title, title))
document.title = title
is_updated = True
if is_updated:
total_updated += 1
document.save()
except Training.DoesNotExist:
logger.debug('Unexisting resource {0}'.format(resource_id))
except Exception:
LoggingHelper.getErrorLogger().error('An error occured while updating {0}'.format(item.id.text),
exc_info=1)
logger.debug('{0} resources updated'.format(total_updated))
def google_apps_add_group_task(group_id, member_id):
logger.debug('Starting Google Apps add to group task')
try:
logger.debug('Get GroupsProvisioningClient')
clientInstance = GroupsProvisioningClient('unishared.com',
OAuthHmacToken('unishared.com', '',
'', '', ACCESS_TOKEN))
clientInstance.ssl = True
logger.debug('Add {0} to {1}'.format(member_id, group_id))
clientInstance.AddMemberToGroup(group_id, member_id)
return True
except RequestError:
LoggingHelper.getErrorLogger().debug("Request error while adding user to Google group", exc_info=1)
return False
except Exception:
LoggingHelper.getErrorLogger().error("Error while adding user to Google group", exc_info=1)
return False
def google_apps_add_group_task(group_id, member_id):
logger.debug('Starting Google Apps add to group task')
try:
logger.debug('Get GroupsProvisioningClient')
clientInstance = GroupsProvisioningClient(
'unishared.com',
OAuthHmacToken('unishared.com', '', '', '', ACCESS_TOKEN))
clientInstance.ssl = True
logger.debug('Add {0} to {1}'.format(member_id, group_id))
clientInstance.AddMemberToGroup(group_id, member_id)
return True
except RequestError:
LoggingHelper.getErrorLogger().debug(
"Request error while adding user to Google group", exc_info=1)
return False
except Exception:
LoggingHelper.getErrorLogger().error(
"Error while adding user to Google group", exc_info=1)
return False
def facebook_task(user, facebook_id, action, params):
try:
user_social = UserSocialAuth.objects.get(user=user, provider='facebook')
access_token = user_social.extra_data['access_token']
graph = get_facebook_graph(access_token=access_token)
if graph:
for n in range(0, 5):
try:
url = None
try:
if u'class' in params:
url = params[u'class']
elif u'link' in params:
url = params[u'link']
if url:
graph.set(u'/', {'ids': url, 'scrape': 'true'})
except OAuthException:
LoggingHelper.getDebugLogger().debug(u'Error while scraping {0}'.format(url), exc_info=1)
return graph.set(u'{0}/{1}'.format(facebook_id, action), params)
except PermissionException:
LoggingHelper.getDebugLogger().debug(
u'Permission not granted (facebook_id : {0}, action {1}, params {2})'.format(facebook_id,
action, params), exc_info=1)
except OAuthException:
LoggingHelper.getDebugLogger().debug(
u'Refreshing token (facebook_id : {0}, action {1}, params {2})'.format(facebook_id, action,
params), exc_info=1)
try:
user_social.refresh_token()
except Exception:
pass
except OpenFacebookException:
LoggingHelper.getErrorLogger().error(
u'Error while posting on Facebook (facebook_id : {0}, action {1}, params {2})'.format(
facebook_id, action, params), exc_info=1)
time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
if action in ['{0}:invite_participate'.format(settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME), '{0}:invite_cowrite'.format(settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME)]:
facebook_task.apply_async([user, facebook_id, '{0}:{1}'.format(settings.FACEBOOK_OPEN_GRAPH_ROOT_NAME, 'cowrite'), params])
break
except UserSocialAuth.DoesNotExist:
LoggingHelper.getDebugLogger().debug('No access token')
def refresh_token(self):
user_social_auth = UserSocialAuth.objects.get(user=self.user, provider='facebook')
params = {
'client_id': settings.FACEBOOK_APP_ID,
'grant_type': 'fb_exchange_token',
'client_secret': settings.FACEBOOK_API_SECRET,
'fb_exchange_token': user_social_auth.extra_data['access_token']
}
try:
response = urlopen(ACCESS_TOKEN + urlencode(params)).read()
# Keys in response are: access_token, expires
response_dict = dict((k, v) for k, v in (e.split('=') for e in response.split('&')))
user_social_auth.extra_data['access_token'] = response_dict['access_token']
user_social_auth.extra_data['expires'] = response_dict['expires']
user_social_auth.save()
except (ValueError, KeyError):
# Error at response['error'] with possible description at
# response['error_description']
LoggingHelper.getErrorLogger().error(u'Error while refreshing access token on Facebook (error : {0}, description : {1})'.format(response['error'], response['error_description']), exc_info=1)
def move_document_to_user_folder(resource_id, user):
logger = LoggingHelper.getDebugLogger()
client = GoogleDriveHelper.get_docs_client()
dest_folder = user.get_profile().drive_folder_id
if not dest_folder:
logger.debug('Creating folder for {0}'.format(user.username))
dest_folder = GoogleDriveHelper.create_unistar_folder(user)
user.get_profile().drive_folder_id = dest_folder
user.get_profile().save()
logger.debug('Moving {0} to {1}'.format(resource_id, GoogleDriveHelper._get_unistar_collection_name(user.username)))
client.files().update(fileId=resource_id, body={"parents":[{'id':dest_folder}]}).execute()
def activation_emails_task():
logger.debug('Starting activation emails task')
users = User.objects.all().select_related('profile')
for user in users:
try:
if user.date_joined and user.last_login:
users_has_documents = Training.objects.filter(creator_id=user).exists()
context = Context({'user': user})
today_at_midnight = timezone.now().replace(hour=0, minute=0, second=0)
logger.debug(user.date_joined)
if timezone.is_aware(user.date_joined):
date_joined = user.date_joined
else:
date_joined = timezone.make_aware(user.date_joined, timezone.get_current_timezone())
days_since_signup = (today_at_midnight - date_joined).days
if days_since_signup == 2 and not users_has_documents:
logger.debug('Sending activation email to {0}'.format(user.email))
context.update({
'ga_campaign_params': 'utm_source=unishared&utm_content=v1&utm_medium=e-mail&utm_campaign=activation_mail'})
email_task.apply_async(
[u'Create your first collaborative document', context, 'activation_mail', [user.email]])
if user.get_profile().last_seen:
days_since_login = (today_at_midnight - user.get_profile().last_seen).days
if days_since_login == 14:
logger.debug('Sending dropping out email to {0}'.format(user.email))
context.update({
'ga_campaign_params': 'utm_source=unishared&utm_content=v1&utm_medium=e-mail&utm_campaign=dropping_out_mail'})
email_task.apply_async([u'Dropping out?', context, 'dropping_out_mail', [user.email]])
except:
LoggingHelper.getErrorLogger().error('An error occurred while sending activation email to user {0}'.format(user.id),
exc_info=1)
def move_document_to_user_folder(resource_id, user):
logger = LoggingHelper.getDebugLogger()
client = GoogleDriveHelper.get_docs_client()
dest_folder = user.get_profile().drive_folder_id
if not dest_folder:
logger.debug('Creating folder for {0}'.format(user.username))
dest_folder = GoogleDriveHelper.create_unistar_folder(user)
user.get_profile().drive_folder_id = dest_folder
user.get_profile().save()
logger.debug('Moving {0} to {1}'.format(
resource_id,
GoogleDriveHelper._get_unistar_collection_name(user.username)))
client.files().update(fileId=resource_id,
body={
"parents": [{
'id': dest_folder
}]
}).execute()
def setUp(self):
super(GoogleDriveHelperTest, self).setUp()
self.logger = LoggingHelper.getDebugLogger()
def live_detection_change_feed(last_result):
logger.debug('Starting live detection with change feed')
client = GoogleDriveHelper.get_docs_client()
params = {'includeDeleted' : False}
if last_result and last_result['startChangeId']:
logger.debug('Last change id : {0}'.format(last_result['startChangeId']))
params['startChangeId'] = last_result['startChangeId']
result = client.changes().list(**params).execute()
task_result = {'startChangeId' : int(result['largestChangeId']) + 1, 'items' : []}
for item in result['items']:
resource_id = item['file']['id']
updated = parser.parse(item['file']['modifiedDate'])
title = item['file']['title']
try:
last_entries = None
if last_result:
last_entries = [last_entry for last_entry in last_result['items'] if last_entry['document_id'] == resource_id]
last_entry = last_entries[0] if last_entries and len(last_entries) > 0 else None
document = Training.objects.get(resource_id=resource_id)
logger.debug('Document recently modified : %s', resource_id)
if document.title != title:
logger.debug('Updating title of document {0}'.format(resource_id))
document.title=title
updated = calendar.timegm(updated.utctimetuple())
document.last_updated = updated
if last_entry:
logger.debug('Previous entry found')
logger.debug('Last entry : {0}'.format(last_entry))
if last_entry['live'] is None:
logger.debug('Enabling live in DB for %s', resource_id)
document.is_live = True
document.is_displayed = True
email_task.apply_async(
['Document live', Context({'document_url': GoogleDriveHelper.get_document_url(resource_id)}),
'document_live', [a[1] for a in settings.ADMINS], None, 'gmail'])
else:
logger.debug('Document already live %s', resource_id)
task_result['items'].append({'document_id': resource_id, 'live': True})
else:
logger.debug('No previous entry found')
logger.debug('Creating entry %s', resource_id)
task_result['items'].append({'document_id': resource_id, 'live': None})
document.save()
except Training.DoesNotExist:
logger.debug('Unexisting resource {0}'.format(resource_id))
except Exception:
LoggingHelper.getErrorLogger().error('An error occurred while detecting live', exc_info=1)
logger.debug('Disabling documents not live anymore')
Training.objects.filter(~Q(resource_id__in=[item['file']['id'] for item in result['items']]), is_live=True).update(is_live=False)
logger.debug('Result : %s', task_result)
live_detection_change_feed.apply_async([task_result], eta=timezone.now()+timedelta(minutes=5))
return task_result
def live_detection_failure(sender, **k):
LoggingHelper.getErrorLogger().error('An error occurred in task {0} with args {1}'.format(sender, sender.request.args), exc_info=1)
if sender is live_detection_change_feed:
with sender.app.connection() as conn:
sender.app.send_task('UniShared_python.website.tasks.live_detection_change_feed', sender.request.args, connection=conn)
from dateutil import parser
from gdata.apps.groups.client import GroupsProvisioningClient
from gdata.client import RequestError
from gdata.gauth import OAuthHmacToken, ACCESS_TOKEN
from open_facebook.exceptions import OAuthException, OpenFacebookException, PermissionException
from social_auth.db.django_models import UserSocialAuth
import redis
from UniShared_python.website.helpers.helpers import GoogleDriveHelper, LoggingHelper, MissingConnectionException
from UniShared_python.website.models import TrainingParticipation, TrainingTempShare
from models import Training
__author__ = 'arnaud'
logger = LoggingHelper.getDebugLogger()
REDIS_CLIENT = redis.Redis.from_url(settings.BROKER_URL)
LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes
def only_one(function=None, timeout=LOCK_EXPIRE):
"""Enforce only one celery task at a time."""
def _dec(run_func):
"""Decorator."""
@wraps(run_func)
def _caller(*args, **kwargs):
"""Caller."""
ret_value = None
have_lock = False
args_list = u','.join([unicode(arg) for arg in args])
def live_detection_change_feed(last_result):
logger.debug('Starting live detection with change feed')
client = GoogleDriveHelper.get_docs_client()
params = {'includeDeleted': False}
if last_result and last_result['startChangeId']:
logger.debug('Last change id : {0}'.format(
last_result['startChangeId']))
params['startChangeId'] = last_result['startChangeId']
result = client.changes().list(**params).execute()
task_result = {
'startChangeId': int(result['largestChangeId']) + 1,
'items': []
}
for item in result['items']:
resource_id = item['file']['id']
updated = parser.parse(item['file']['modifiedDate'])
title = item['file']['title']
try:
last_entries = None
if last_result:
last_entries = [
last_entry for last_entry in last_result['items']
if last_entry['document_id'] == resource_id
]
last_entry = last_entries[0] if last_entries and len(
last_entries) > 0 else None
document = Training.objects.get(resource_id=resource_id)
logger.debug('Document recently modified : %s', resource_id)
if document.title != title:
logger.debug(
'Updating title of document {0}'.format(resource_id))
document.title = title
updated = calendar.timegm(updated.utctimetuple())
document.last_updated = updated
if last_entry:
logger.debug('Previous entry found')
logger.debug('Last entry : {0}'.format(last_entry))
if last_entry['live'] is None:
logger.debug('Enabling live in DB for %s', resource_id)
document.is_live = True
document.is_displayed = True
email_task.apply_async([
'Document live',
Context({
'document_url':
GoogleDriveHelper.get_document_url(resource_id)
}), 'document_live', [a[1] for a in settings.ADMINS],
None, 'gmail'
])
else:
logger.debug('Document already live %s', resource_id)
task_result['items'].append({
'document_id': resource_id,
'live': True
})
else:
logger.debug('No previous entry found')
logger.debug('Creating entry %s', resource_id)
task_result['items'].append({
'document_id': resource_id,
'live': None
})
document.save()
except Training.DoesNotExist:
logger.debug('Unexisting resource {0}'.format(resource_id))
except Exception:
LoggingHelper.getErrorLogger().error(
'An error occurred while detecting live', exc_info=1)
logger.debug('Disabling documents not live anymore')
Training.objects.filter(
~Q(resource_id__in=[item['file']['id'] for item in result['items']]),
is_live=True).update(is_live=False)
logger.debug('Result : %s', task_result)
live_detection_change_feed.apply_async([task_result],
eta=timezone.now() +
timedelta(minutes=5))
return task_result
def setUp(self):
super(GoogleDriveHelperTest, self).setUp()
self.logger = LoggingHelper.getDebugLogger()
from celery import task
from dateutil import parser
from gdata.apps.groups.client import GroupsProvisioningClient
from gdata.client import RequestError
from gdata.gauth import OAuthHmacToken, ACCESS_TOKEN
from open_facebook.exceptions import OAuthException, OpenFacebookException, PermissionException
from social_auth.db.django_models import UserSocialAuth
import redis
from UniShared_python.website.helpers.helpers import GoogleDriveHelper, LoggingHelper, MissingConnectionException
from UniShared_python.website.models import TrainingParticipation, TrainingTempShare
from models import Training
__author__ = 'arnaud'
logger = LoggingHelper.getDebugLogger()
REDIS_CLIENT = redis.Redis.from_url(settings.BROKER_URL)
LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes
def only_one(function=None, timeout=LOCK_EXPIRE):
"""Enforce only one celery task at a time."""
def _dec(run_func):
"""Decorator."""
@wraps(run_func)
def _caller(*args, **kwargs):
"""Caller."""
ret_value = None
have_lock = False
args_list = u','.join([unicode(arg) for arg in args])