from django.db.models import Q import gviz_api import datetime as dt from django.http import HttpResponse from snh.models.twittermodel import * from snh.models.facebookmodel import * from snh.models.youtubemodel import * from snh.models.dailymotionmodel import * from snh.utils import get_datatables_records, Twitter_raw_json_posts_data,generate_csv_response, twUserAjaxTableError from datetime import datetime import snhlogger logger = snhlogger.init_logger(__name__, "view.log") import time import types import re import csv, codecs, cStringIO import json from settings import DEBUGCONTROL, dLogger debugging = DEBUGCONTROL['twitterview'] if debugging: print "DEBBUGING ENABLED IN %s"%__name__ # # TWITTER # tw_status_fields = [['fid','text'],
from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from cronharvester import facebookch, youtubech, twitterch import snhlogger logger = snhlogger.init_logger(__name__, "ElasticSearch.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="cronES") try: logger.info("Will archive data into ElasticSearch") twitterch.esArchive() #facebookch.esArchive() #youtubech.esArchive() except: msg = u"Highest exception for the ElasticSearch cron. Not good." logger.exception(msg) logger.info("The archiving into the ElasticSearch has ended.")
from elasticsearch import Elasticsearch import snhlogger import os from settings import DEBUGCONTROL, dLogger debugging = DEBUGCONTROL['twitterch'] if debugging: print "DEBBUGING ENABLED IN %s" % __name__ import requests from requests_oauthlib import OAuth1 requests.packages.urllib3.disable_warnings() logger = snhlogger.init_logger(__name__, "twitter.log") @dLogger.debug def run_twitter_harvester(): if debugging: dLogger.log("run_twitter_harvester()") #custom_export() #return harvester_list = sort_harvesters_by_priority( TwitterHarvester.objects.all()) for harvester in harvester_list: harvester.harvest_in_progress = False harvester.save()
# coding=UTF-8 from collections import deque from datetime import datetime import time from httplib2 import Http from urllib import urlencode import json import urllib2 from django.db import models from snh.models.common import * import snhlogger logger = snhlogger.init_logger(__name__, "dailymotion.log") class DailyMotionHarvester(AbstractHaverster): harvester_type = "DailyMotion" base = 'https://api.dailymotion.com/json' oauth = 'https://api.dailymotion.com/oauth/token' key = models.CharField(max_length=255, null=True) secret = models.CharField(max_length=255, null=True) user = models.CharField(max_length=255, null=True) password = models.CharField(max_length=255, null=True) access_token = None refresh_token = None
# coding=UTF-8 import os import subprocess from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.models.dailymotionmodel import * from settings import MEDIA_ROOT import snhlogger logger = snhlogger.init_logger(__name__, "dailymotion_downloader.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="crondm_dl") try: logger.info("Will run the dailymotion video downloader.") videos = DMVideo.objects.all() for vid in videos: logger.info("Video: %s" % vid.url) logger.info("User: %s" % vid.user) userfid = vid.user.fid
# coding=UTF-8 from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.management.commands.cronharvester import youtubech import snhlogger logger = snhlogger.init_logger(__name__, "youtube.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="cronyt") try: logger.info("Will run the Youtube harvesters.") youtubech.run_youtube_harvester() except: msg = u"Highest exception for the youtube cron. Not good." logger.exception(msg) logger.info("The harvest has end for the Youtube harvesters.")
# coding=UTF-8 from datetime import timedelta import resource import time from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from snh.models.dailymotionmodel import * import snhlogger logger = snhlogger.init_logger(__name__, "dailymotion.log") from settings import DEBUGCONTROL debugging = DEBUGCONTROL['dailymotionch'] if DEBUGCONTROL['dailymotionch']: print "DEBBUGING ENABLED IN %s" % __name__ debugLogger = snhlogger.init_custom_logger( 'debug' + __name__, "debugLogger.log", '(%(filename)-17s) %(message)s') def run_dailymotion_harvester(): harvester_list = DailyMotionHarvester.objects.all() for harvester in harvester_list: logger.info(u"The harvester %s is %s" % (unicode(harvester), "active" if harvester.is_active else "inactive")) if harvester.is_active: run_harvester_v1(harvester) def sleeper(retry_count):
from django.db.models import Q from django.template.loader import render_to_string from django.http import HttpResponse from django.utils.cache import add_never_cache_headers from django.utils import simplejson import csv, codecs, cStringIO import snhlogger logger = snhlogger.init_logger(__name__, "view.log") class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = cStringIO.StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() def writerow(self, row): self.writer.writerow([s.encode("utf-8") for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream
# coding=UTF-8 import sys import time import Queue import threading import urlparse import resource from django.core.exceptions import ObjectDoesNotExist from facepy.exceptions import FacepyError from fandjango.models import User as FanUser from snh.models.facebookmodel import * import snhlogger logger = snhlogger.init_logger(__name__, "facebook.log") E_DNEXIST = "E_DNEXIST" E_PATH = "E_PATH" E_FALSE = "E_FALSE" E_UNMAN = "E_UNMAN" E_UNEX = "E_UNEX" E_USER_QUOTA = "E_USER_QUOTA" E_QUOTA = "E_QUOTA" E_GRAPH = "E_GRAPH" E_MAX_RETRY = "E_MAX_RETRY" E_CRITICALS = [E_DNEXIST, E_PATH, E_FALSE, E_UNMAN, E_USER_QUOTA] def run_facebook_harvester(): harvester_list = FacebookHarvester.objects.all()
# coding=UTF-8 from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.management.commands.cronharvester import twitterch import snhlogger logger = snhlogger.init_logger(__name__, "twitter.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="crontw") try: logger.info("Will run the Twitter harvesters.") twitterch.run_twitter_harvester() except: msg = u"Highest exception for the twitter cron. Not good." logger.exception(msg) logger.info("The harvest has end for the Twitter harvesters.")
import urlparse #import psutil import datetime import facebook import json from DebugLogger import DebugLogger from django.core.exceptions import ObjectDoesNotExist from facepy.exceptions import FacepyError from fandjango.models import User as FanUser, OAuthToken from fandjango.decorators import * from snh.models.facebookmodel import * from django.core import serializers import snhlogger logger = snhlogger.init_logger(__name__, "facebook.log") logger.info(' ' * 400) from settings import DEBUGCONTROL, dLogger, FACEBOOK_APPLICATION_ID, FACEBOOK_APPLICATION_SECRET_KEY debugging = DEBUGCONTROL['facebookch'] if debugging: print "DEBBUGING ENABLED IN %s" % __name__ E_DNEXIST = "E_DNEXIST" E_PATH = "E_PATH" E_FALSE = "E_FALSE" E_UNMAN = "E_UNMAN" E_UNEX = "E_UNEX" E_USER_QUOTA = "E_USER_QUOTA" E_QUOTA = "E_QUOTA" E_GRAPH = "E_GRAPH" E_MAX_RETRY = "E_MAX_RETRY"
# coding=UTF-8 import os import subprocess from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.models.youtubemodel import * from settings import MEDIA_ROOT import snhlogger logger = snhlogger.init_logger(__name__, "youtube_downloader.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="cronyt_dl") try: logger.info("Will run the youtube video downloader.") videos = YTVideo.objects.all() for vid in videos: logger.info("Video: %s" % vid.swf_url) logger.info("User: %s" % vid.user)
# coding=UTF-8 from datetime import timedelta import resource import time import urllib from django.core.exceptions import ObjectDoesNotExist from snh.models.youtubemodel import * from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned import snhlogger logger = snhlogger.init_logger(__name__, "youtube.log") def run_youtube_harvester(): harvester_list = YoutubeHarvester.objects.all() for harvester in harvester_list: logger.info(u"The harvester %s is %s" % (unicode(harvester), "active" if harvester.is_active else "inactive")) if harvester.is_active: run_harvester_v1(harvester) def sleeper(retry_count): retry_delay = 1 wait_delay = retry_count*retry_delay wait_delay = 10 if wait_delay > 10 else wait_delay time.sleep(wait_delay) def get_timedelta(dm_time): ts = datetime.strptime(dm_time,'%Y-%m-%dT%H:%M:%S+0000')
# coding=UTF-8 import os import subprocess from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.models.dailymotionmodel import * from settings import MEDIA_ROOT import snhlogger logger = snhlogger.init_logger(__name__, "dailymotion_downloader.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="crondm_dl") try: logger.info("Will run the dailymotion video downloader.") videos = DMVideo.objects.all() for vid in videos: logger.info("Video: %s" % vid.url) logger.info("User: %s" % vid.user)
# coding=UTF-8 import os import subprocess from tendo import singleton from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ObjectDoesNotExist from snh.models.youtubemodel import * from settings import MEDIA_ROOT import snhlogger logger = snhlogger.init_logger(__name__, "youtube_downloader.log") class Command(BaseCommand): #args = '<poll_id poll_id ...>' #help = 'Closes the specified poll for voting' def handle(self, *args, **options): me = singleton.SingleInstance(flavor_id="cronyt_dl") try: logger.info("Will run the youtube video downloader.") videos = YTVideo.objects.all() for vid in videos: logger.info("Video: %s" % vid.swf_url) logger.info("User: %s" % vid.user) userfid = vid.user.fid