from django.db.models import Q import gviz_api import datetime as dt from django.http import HttpResponse from snh.models.twittermodel import * from snh.models.facebookmodel import * from snh.models.youtubemodel import * from snh.models.dailymotionmodel import * from snh.utils import get_datatables_records, Twitter_raw_json_posts_data,generate_csv_response, twUserAjaxTableError from datetime import datetime import snhlogger logger = snhlogger.init_logger(__name__, "view.log") import time import types import re import csv, codecs, cStringIO import json from settings import DEBUGCONTROL, dLogger debugging = DEBUGCONTROL['twitterview'] if debugging: print "DEBBUGING ENABLED IN %s"%__name__ # # TWITTER # tw_status_fields = [['fid','text'],
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from cronharvester import facebookch, youtubech, twitterch
import snhlogger
logger = snhlogger.init_logger(__name__, "ElasticSearch.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="cronES")
try:
logger.info("Will archive data into ElasticSearch")
twitterch.esArchive()
#facebookch.esArchive()
#youtubech.esArchive()
except:
msg = u"Highest exception for the ElasticSearch cron. Not good."
logger.exception(msg)
logger.info("The archiving into the ElasticSearch has ended.")
from elasticsearch import Elasticsearch
import snhlogger
import os
from settings import DEBUGCONTROL, dLogger
debugging = DEBUGCONTROL['twitterch']
if debugging: print "DEBBUGING ENABLED IN %s" % __name__
import requests
from requests_oauthlib import OAuth1
requests.packages.urllib3.disable_warnings()
logger = snhlogger.init_logger(__name__, "twitter.log")
@dLogger.debug
def run_twitter_harvester():
if debugging: dLogger.log("run_twitter_harvester()")
#custom_export()
#return
harvester_list = sort_harvesters_by_priority(
TwitterHarvester.objects.all())
for harvester in harvester_list:
harvester.harvest_in_progress = False
harvester.save()
# coding=UTF-8
from collections import deque
from datetime import datetime
import time
from httplib2 import Http
from urllib import urlencode
import json
import urllib2
from django.db import models
from snh.models.common import *
import snhlogger
logger = snhlogger.init_logger(__name__, "dailymotion.log")
class DailyMotionHarvester(AbstractHaverster):
harvester_type = "DailyMotion"
base = 'https://api.dailymotion.com/json'
oauth = 'https://api.dailymotion.com/oauth/token'
key = models.CharField(max_length=255, null=True)
secret = models.CharField(max_length=255, null=True)
user = models.CharField(max_length=255, null=True)
password = models.CharField(max_length=255, null=True)
access_token = None
refresh_token = None
# coding=UTF-8
import os
import subprocess
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.models.dailymotionmodel import *
from settings import MEDIA_ROOT
import snhlogger
logger = snhlogger.init_logger(__name__, "dailymotion_downloader.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="crondm_dl")
try:
logger.info("Will run the dailymotion video downloader.")
videos = DMVideo.objects.all()
for vid in videos:
logger.info("Video: %s" % vid.url)
logger.info("User: %s" % vid.user)
userfid = vid.user.fid
# coding=UTF-8
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.management.commands.cronharvester import youtubech
import snhlogger
logger = snhlogger.init_logger(__name__, "youtube.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="cronyt")
try:
logger.info("Will run the Youtube harvesters.")
youtubech.run_youtube_harvester()
except:
msg = u"Highest exception for the youtube cron. Not good."
logger.exception(msg)
logger.info("The harvest has end for the Youtube harvesters.")
# coding=UTF-8
from datetime import timedelta
import resource
import time
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from snh.models.dailymotionmodel import *
import snhlogger
logger = snhlogger.init_logger(__name__, "dailymotion.log")
from settings import DEBUGCONTROL
debugging = DEBUGCONTROL['dailymotionch']
if DEBUGCONTROL['dailymotionch']:
print "DEBBUGING ENABLED IN %s" % __name__
debugLogger = snhlogger.init_custom_logger(
'debug' + __name__, "debugLogger.log", '(%(filename)-17s) %(message)s')
def run_dailymotion_harvester():
harvester_list = DailyMotionHarvester.objects.all()
for harvester in harvester_list:
logger.info(u"The harvester %s is %s" %
(unicode(harvester),
"active" if harvester.is_active else "inactive"))
if harvester.is_active:
run_harvester_v1(harvester)
def sleeper(retry_count):
from django.db.models import Q
from django.template.loader import render_to_string
from django.http import HttpResponse
from django.utils.cache import add_never_cache_headers
from django.utils import simplejson
import csv, codecs, cStringIO
import snhlogger
logger = snhlogger.init_logger(__name__, "view.log")
class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
# Redirect output to a queue
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
self.encoder = codecs.getincrementalencoder(encoding)()
def writerow(self, row):
self.writer.writerow([s.encode("utf-8") for s in row])
# Fetch UTF-8 output from the queue ...
data = self.queue.getvalue()
data = data.decode("utf-8")
# ... and reencode it into the target encoding
data = self.encoder.encode(data)
# write to the target stream
# coding=UTF-8
import sys
import time
import Queue
import threading
import urlparse
import resource
from django.core.exceptions import ObjectDoesNotExist
from facepy.exceptions import FacepyError
from fandjango.models import User as FanUser
from snh.models.facebookmodel import *
import snhlogger
logger = snhlogger.init_logger(__name__, "facebook.log")
E_DNEXIST = "E_DNEXIST"
E_PATH = "E_PATH"
E_FALSE = "E_FALSE"
E_UNMAN = "E_UNMAN"
E_UNEX = "E_UNEX"
E_USER_QUOTA = "E_USER_QUOTA"
E_QUOTA = "E_QUOTA"
E_GRAPH = "E_GRAPH"
E_MAX_RETRY = "E_MAX_RETRY"
E_CRITICALS = [E_DNEXIST, E_PATH, E_FALSE, E_UNMAN, E_USER_QUOTA]
def run_facebook_harvester():
harvester_list = FacebookHarvester.objects.all()
# coding=UTF-8
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.management.commands.cronharvester import twitterch
import snhlogger
logger = snhlogger.init_logger(__name__, "twitter.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="crontw")
try:
logger.info("Will run the Twitter harvesters.")
twitterch.run_twitter_harvester()
except:
msg = u"Highest exception for the twitter cron. Not good."
logger.exception(msg)
logger.info("The harvest has end for the Twitter harvesters.")
import urlparse
#import psutil
import datetime
import facebook
import json
from DebugLogger import DebugLogger
from django.core.exceptions import ObjectDoesNotExist
from facepy.exceptions import FacepyError
from fandjango.models import User as FanUser, OAuthToken
from fandjango.decorators import *
from snh.models.facebookmodel import *
from django.core import serializers
import snhlogger
logger = snhlogger.init_logger(__name__, "facebook.log")
logger.info(' ' * 400)
from settings import DEBUGCONTROL, dLogger, FACEBOOK_APPLICATION_ID, FACEBOOK_APPLICATION_SECRET_KEY
debugging = DEBUGCONTROL['facebookch']
if debugging: print "DEBBUGING ENABLED IN %s" % __name__
E_DNEXIST = "E_DNEXIST"
E_PATH = "E_PATH"
E_FALSE = "E_FALSE"
E_UNMAN = "E_UNMAN"
E_UNEX = "E_UNEX"
E_USER_QUOTA = "E_USER_QUOTA"
E_QUOTA = "E_QUOTA"
E_GRAPH = "E_GRAPH"
E_MAX_RETRY = "E_MAX_RETRY"
# coding=UTF-8
import os
import subprocess
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.models.youtubemodel import *
from settings import MEDIA_ROOT
import snhlogger
logger = snhlogger.init_logger(__name__, "youtube_downloader.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="cronyt_dl")
try:
logger.info("Will run the youtube video downloader.")
videos = YTVideo.objects.all()
for vid in videos:
logger.info("Video: %s" % vid.swf_url)
logger.info("User: %s" % vid.user)
# coding=UTF-8
from datetime import timedelta
import resource
import time
import urllib
from django.core.exceptions import ObjectDoesNotExist
from snh.models.youtubemodel import *
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
import snhlogger
logger = snhlogger.init_logger(__name__, "youtube.log")
def run_youtube_harvester():
harvester_list = YoutubeHarvester.objects.all()
for harvester in harvester_list:
logger.info(u"The harvester %s is %s" %
(unicode(harvester),
"active" if harvester.is_active else "inactive"))
if harvester.is_active:
run_harvester_v1(harvester)
def sleeper(retry_count):
retry_delay = 1
wait_delay = retry_count*retry_delay
wait_delay = 10 if wait_delay > 10 else wait_delay
time.sleep(wait_delay)
def get_timedelta(dm_time):
ts = datetime.strptime(dm_time,'%Y-%m-%dT%H:%M:%S+0000')
# coding=UTF-8
import os
import subprocess
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.models.dailymotionmodel import *
from settings import MEDIA_ROOT
import snhlogger
logger = snhlogger.init_logger(__name__, "dailymotion_downloader.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="crondm_dl")
try:
logger.info("Will run the dailymotion video downloader.")
videos = DMVideo.objects.all()
for vid in videos:
logger.info("Video: %s" % vid.url)
logger.info("User: %s" % vid.user)
# coding=UTF-8
import os
import subprocess
from tendo import singleton
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from snh.models.youtubemodel import *
from settings import MEDIA_ROOT
import snhlogger
logger = snhlogger.init_logger(__name__, "youtube_downloader.log")
class Command(BaseCommand):
#args = '<poll_id poll_id ...>'
#help = 'Closes the specified poll for voting'
def handle(self, *args, **options):
me = singleton.SingleInstance(flavor_id="cronyt_dl")
try:
logger.info("Will run the youtube video downloader.")
videos = YTVideo.objects.all()
for vid in videos:
logger.info("Video: %s" % vid.swf_url)
logger.info("User: %s" % vid.user)
userfid = vid.user.fid
