__author__ = 'katharina hafner'

# 27/12/15: crawling website moviebarcodes.tumblr.com/movie-index;

# building/ testing RE

# 03/01/-04/01/16: introduction mongodb; storing simple data in Collection

# 'movie'; establish connection to original image files;

# storing image files in gridfs bucket with post_id as ref-key

# 09/01/16 get metadata from omdb-api by title (laura)

# get metadata from omdb-api by id ( scrap tt_id from website

# http://moviebarcode.tumble.com/post/xxxxxxxx); if there's no

# hit, search film-metadata by title

# 19,20,21/01/16 html-code of our source-website moviebarcodes.tumblr.com

# changed -> fitting code

# 23/01/16 get dominant colors with ColorThief from each imagefile

# store in collections

# 27/01/16 fetching posters from omdb, storing at grid fs; changes

# color thief

# 29/01/16 color clustering data into mongodb

# 02/02/16 restrict hits by removing Titles with "...[Sequence from]"

# 19/02/16 get subtitles from OpenSubtitles.org

# 21/02/16 insert registered UserAgent account (OpenSubtitles) + first try to format subtitles

# 22/02/16 format subtitle string (remove unnecessary characters and numbers)

# 24/02/16 select subtitles with the best ratings

# 02/03/16 coordinate requests to www.opensubtitles.com

import urllib.parse

from urllib import parse

import urllib.request

from urllib.request import urlopen

import urllib.error

import urllib.response

import requests

import re # Regular Expressions

from pymongo import MongoClient # NoSQL DB Framework

import gridfs # Mongo DB Grid FS Bucket

import json

# import colorthief as ct # get dominant colors from image

# import webcolors # conversation rgb to hex

from Naked.toolshed.shell import muterun_js

import numpy as np

from PIL import Image

from colormap import rgb2hex

import cv2

from sklearn.cluster import KMeans

import io # get subtitles from OpenSubtitles

import gzip # unzip subtitles

import base64 # get subtitles from OpenSubtitles, decode base64

from xmlrpc.client import ServerProxy

try:

from html import unescape # python 3.4+

except ImportError:

try:

from html.parser import HTMLParser # python 3.x (<3.4)

except ImportError:

from HTMLParser import HTMLParser # python 2.x

unescape = HTMLParser().unescape

from textProcessing import lemmatize

def get_content(link):

return str_response

def process_file(file):

print('\n')

def urlEncodeNonAscii(b):

return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

def get_movie_json(title):

return obj

def get_movie_json_by_id(imdb_id):

response = urlopen("http://www.omdbapi.com/?i=" + imdb_id).read().decode('utf8')

obj = json.loads(response)

return obj

def objdata_to_db(obj):

l_type, l_rated, l_poster

def fill_collection(db, fs, l_post_id, l_imdbid, l_title, l_year, l_image, l_actors,

print('******************************Exception beim fillcollection()', e)

def centroid_histogram(clt):

return hist

def get_dominant_colors_by_colordiff(db, fs, l_post_id):

update_value_in_db(db, l_post_id, "dominantColors", dominant_colors)

def get_subtitles(db, l_post_id, l_imdbid):

print("No subtitle available")

# get SubtitleID with the best rating

def get_best_subtitle(array):

return best_subtitle

def update_value_in_db(db, l_post_id, key, value):

print("no entry for ", l_post_id)

# removes numbers, special characters, control characters and tags

def remove_invalid_characters(decoded_subtitle):

return subtitles_string

# commandline output katharina // ONLY TEST!

def debugKH(s):

print("KH:", s)

def main():

link = "http://moviebarcode.tumblr.com/movie-index"

get_content(link)

if __name__ == '__main__':

main()