def parse_authors(self):
# Create authors
print "Parsing Authors..."
f = open(data_io.get_paths()["author_processed_path"], "r")
titles = f.readline()
for l in f.readlines():
res = l.strip().split(",")
# Titles
raw_title = unidecode.unidecode(unicode(res[1], encoding="utf-8"))
(name, surname) = nlp.filter_title(raw_title)
try:
self.surnames[surname] = self.surnames[surname] + 1
except:
self.surnames[surname] = 1
#Affiliations
raw_affiliation = unidecode.unidecode(
unicode(res[2], encoding="utf-8"))
affiliation = nlp.filter_affiliation(raw_affiliation)
try:
self.affiliations[
affiliation] = self.affiliations[affiliation] + 1
except:
self.affiliations[affiliation] = 1
self.authors[int(res[0])] = author.Author(int(res[0]), name,
surname, affiliation)
print "Done"
f.close()
def parse_db(self):
conn = data_io.get_db_conn()
cursor = conn.cursor()
# Create authors
print "Parsing Authors..."
cursor.execute("SELECT * from Author;")
for res in cursor:
self.authors[res[0]] = author.Author(res[0], res[1], res[2])
print "Done"
# Create Papers
print "Parsing Papers..."
cursor.execute("SELECT * from Paper;")
for res in cursor:
self.papers[res[0]] = paper.Paper(res[0], res[1], res[2], res[3],
res[4], res[5])
print "Done"
# First Update all journal/conference/coauthor information
print "Parsing PaperAuthors..."
cursor.execute("SELECT * from PaperAuthor;")
for res in cursor:
paper_id = res[0]
author_id = res[1]
curr_author = None
curr_paper = None
if paper_id in self.papers.keys():
curr_paper = self.papers[paper_id]
if author_id in self.authors.keys():
curr_author = self.authors[author_id]
self.update_paperauthor(curr_paper, curr_author, author_id)
print "Done"
def collect_authors(self):
author_files = self.get_files(os.path.join(dirs.src, dirs.src_author),
'')
self.authors = {}
for a in author_files:
a = author.Author(a)
self.authors[a.shortname] = a
def test_add_author(self):
prepare_db_file()
db = author_database.AuthorDataBase('adb_test.json')
self.assertFalse(db.is_author_in_db("7"))
aut = author.Author("7", "Test", "User")
self.assertTrue(db.add_author(aut))
self.assertTrue(db.is_author_in_db("7"))
remove_db_file()
def gather_data(user_id_list, limit=5):
"Return an Author object for the users in the provided list. Gather [limit] messages from each one."
for user in user_id_list:
if sdb.get_user_tweets_count(user) > limit:
print user
yield author.Author(sdb.get_user_info(user),
sdb.get_user_tweets(user, limit))
else:
print "{} , {} , {}".format(user, "na", datetime.datetime.now())
def attachWorkToAuthor(workName, date, authorName):
authorInstance = dynamicobjectkeeper.getAuthor(authorName)
if authorInstance == None:
authorInstance = author.Author(authorName)
dynamicobjectkeeper.pushAuthor(authorInstance)
workInstance = work.Work(workName, date)
authorInstance.attachWork(workInstance)
return
def load_author(self):
try:
with open(self.load_directory, 'r') as file:
# logger : loading data from file
x = json.load(file)
temp_dict = dict()
for i in x:
temp_dict[i] = author.Author(i, x[i]['name'],
x[i]['surname'])
return temp_dict
except FileNotFoundError:
# logger : file does not exist
return dict()
def main():
rev = review.Review("Horosho")
rev2 = review.Review("Ploho")
autor_Tolstoy = author.Author("Leo", "Tolstoy", 1828)
boo = book.Book(autor_Tolstoy, "WAR Pease", "1488", "War")
boo1 = book.Book(autor_Tolstoy, "Anna K", "1489", "Drama")
boo.add_review(rev)
boo.add_review(rev2)
autor_Tolstoy.add_book(boo)
autor_Tolstoy.add_book(boo1)
print(autor_Tolstoy)
print(boo)
print(boo1)
def __init__(self, book_dict):
for key, val in book_dict.items():
if key == 'authors':
authors = []
for author_dict in book_dict[key].values():
authors.append(author.Author(author_dict))
self.__dict__[key] = authors
continue
if key == 'similar_books':
sim_books = []
for sim_book_list in book_dict[key].values():
for sim_book_dict in sim_book_list:
sim_books.append(similar_book.SimilarBook(sim_book_dict))
self.__dict__[key] = sim_books
continue
self.__dict__[key] = val
def add_author(stdscr, adb: ab.AuthorDataBase):
h, w = stdscr.getmaxyx()
stdscr.clear()
stdscr.addstr(0, w // 2 - len("Name") // 2, "Name")
name = read_word(stdscr, h, w)
stdscr.clear()
stdscr.addstr(0, w // 2 - len("Surname") // 2, "Surname")
surname = read_word(stdscr, h, w)
aut_id = int(adb.get_max_id()) + 1
au = aut.Author(str(aut_id), name, surname)
if adb.add_author(au):
adb.save_author()
print_center(stdscr, "Success - press key to return")
stdscr.getch()
else:
print_center(stdscr, "Error - press key to return")
stdscr.getch()
def stream():
#track = ['#targets', '#iceisis', '#opiceisis']
account = 4
authKit = author.Author(account)
track = ['Targeted IS accounts']
follow = [
'3012875395', '3012814332', '82415925', '3425725672', '4720033461',
'3428485468', '3063852029', '3066135370'
]
reload(sys)
sys.setdefaultencoding('utf-8')
name = datetime.datetime.strftime(datetime.datetime.now(),
'luckytroll%Y.%m.%d.%H.%M.%S')
listen = SListener(authKit.api, "reports/" + name)
stream = tweepy.Stream(authKit.auth, listen)
print "Streaming started..."
try:
stream.filter(track=track, follow=follow)
except:
print "error!"
stream.disconnect()
def load_nodes(graph):
with open(AUTHORS_FILE, 'r') as f:
line = f.readline()
while line != "":
line = line[len("#index "):] # remove index prefix from line
a_id = line.strip() # 1st line (stripped) is author's index
line = f.readline() # read next line
line = line[len("#n "):] # remove name prefix from line
if line.strip() == "": # make sure name isn't empty
line = "<empty_" + a_id + ">"
name = line.strip() # 2nd line (stripped) is author's name
auth = author.Author(int(a_id), name) # create author and
graph.add_node(auth) # add it to the graph
#if graph.number_of_nodes() % 100000 == 0:
# print graph.number_of_nodes()
while line.strip(
) != "": # ignore additional author details and read next line
line = f.readline()
line = f.readline()
from slistener import SListener
import time, tweepy, sys
from tweepy import OAuthHandler
import author
import sys
import time
import datetime
account = 2
authKit = author.Author(account)
def stream():
#track = ['#targets', '#iceisis', '#opiceisis']
account = 4
authKit = author.Author(account)
track = ['Targeted IS accounts']
follow = [
'3012875395', '3012814332', '82415925', '3425725672', '4720033461',
'3428485468', '3063852029', '3066135370'
]
reload(sys)
sys.setdefaultencoding('utf-8')
name = datetime.datetime.strftime(datetime.datetime.now(),
'luckytroll%Y.%m.%d.%H.%M.%S')
listen = SListener(authKit.api, "reports/" + name)
stream = tweepy.Stream(authKit.auth, listen)
print "Streaming started..."
try:
import basket
import discount
import tkinter as tk
import appSettings
from tkinter import ttk
from pages import *
set = appSettings.Settings
discount1 = discount.Discount("NONE")
container = None
buttonframe = None
mainBook = None
#authors
author1 = author.Author("Kristie", "Roaling", 1, (2000, 13, 14))
author2 = author.Author("Din", "Bay", 2, (2000, 13, 14))
# books
book1 = book.Book("Harry Potter", 300, 8888, 50, "Fantasy", "BelPechat",
author1, False, "res/ava1.gif")
book2 = book.Book("Barry Potter P2", 400, 8889, 100, "Fantasy", "BelPechat",
author1, False, "res/ava3.png")
book3 = book.Book("Piter Pan", 210, 1111, 200, "Fantasy", "RusPechat", author2,
False, "res/hpGF.png")
book4 = book.Book("Aiter Pan P2", 390, 1111, 99, "Fantasy", "RusPechat",
author2, False, "res/hpGF.png")
book5 = book.Book("Piter Pan", 210, 1111, 200, "Fantasy", "RusPechat", author2,
False, "res/hpGF.png")
book6 = book.Book("Piter Pan", 210, 1111, 200, "Fantasy", "RusPechat", author2,
False, "res/hpGF.png")
def test_author_init(self):
aut = author.Author("0", "AuthorsName", "AuthorsSurname")
self.assertEqual(aut.uid, "0")
self.assertEqual(aut.aut_name, "AuthorsName")
self.assertEqual(aut.aut_surname, "AuthorsSurname")
import book
import author
first_author = author.Author("John", "Doe", "USA")
book = book.Book("All about Python", 1, author)
book.lend("testUser")
author_id = 1
author_list = list()
while flag:
print("Добавление автора. Введите имя:")
name = input()
print("Введите возраст:")
age = int(input())
print("Введите пол (Male/Female):")
gender = input()
print("Введите e-mail автора:")
email = input()
# Занесение в бд? Или в лист?
new_author = author.Author(author_id, name, email, gender, age)
author_id += 1
author_list.append(new_author)
print("Автор:", new_author, "добавлен")
print("Добавить еще автора? (Y/N)")
author_editor_exit = input()
if author_editor_exit == "N":
flag = False
# Создаем доп авторов в конструкторе
Nattzy = author.Author(2, "Лимонов Роман", "*****@*****.**", "Male", 24)
Vorxal = author.Author(3, "Воронков Алексей", "*****@*****.**", "Male", 21)
author_list.append(Nattzy)
author_list.append(Vorxal)
def test_author_prepare_to_json(self):
aut = author.Author("0", "AuthorsName", "AuthorsSurname")
dic = aut.prepare_to_json()
self.assertEqual(dic['name'], "AuthorsName")
self.assertEqual(dic['surname'], "AuthorsSurname")
def getUsers(catalogue="_rawtweets_3_1_users.csv",
outfile="resultAccounts_full24.json",
key=""):
picaddress = key
if not os.path.exists(picaddress):
os.makedirs(picaddress)
f = open(catalogue, "r")
l = f.readline()
print(l)
authKit = author.Author(1)
counter = 0
while (l != ''):
items = l.split(',')
created = items[0]
created_short = items[1]
#created ="3\01\2016"
id = items[2].split('\n')[0]
bot = items[3]
bot_timezone = items[4].split('\n')[0]
print(id)
try:
id = int(id)
except:
print(id)
print("Error - ID non-integer")
l = f.readline()
continue
try:
user = authKit.api.get_user(id)
d = user.__dict__
created_at = d['created_at']
createdat = created_at.strftime("%c")
createdat_short = created_at.strftime("%m.%Y")
location = d['location'].encode('utf-8')
description = d['description'].encode('utf-8')
description = description.replace('^', " ")
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
now_short = datetime.now().strftime('%d.%m.%Y')
#desc = description['description'].replace("\n", " | ")
dic = {
'time_now_chicago': now,
'time_now_chicago_short': now_short,
'report_created': created,
'report_created_short': created_short,
'bot': bot,
'bot_timezone': bot_timezone,
'source_id': id,
'exists': True,
'has_extended_profile': d['has_extended_profile'],
'description': description,
'profile_use_background_image':
d['profile_use_background_image'],
'time_zone': d['time_zone'],
'verified': d['verified'],
'followers_count': d['followers_count'],
'geo_enabled': d['geo_enabled'],
'lang': d['lang'],
'statuses_count': d['statuses_count'],
'friends_count': d['friends_count'],
'favourites_count': d['favourites_count'],
'account_created': createdat,
'account_created_short': createdat_short,
'location': location,
'default_profile': d['default_profile'],
'profile_image_url': d['profile_image_url'],
'screen_name': d['screen_name']
}
ids = str(id)
picname = os.path.join(picaddress, ids + ".jpeg")
address = d['profile_image_url']
urllib.urlretrieve(address, picname)
except tweepy.TweepError as e:
try:
print(e.message[0]['code'])
if (e.message[0]['code'] == 88):
if (authKit.account < 3):
authKit = author.Author(authKit.account + 1)
print("account:")
print(authKit.account)
continue
else:
print("Going to sleep")
time.sleep(10 * 60)
authKit = author.Author(1)
continue
except:
print("This is weird...But I am going on")
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
now_short = datetime.now().strftime('%d.%m.%Y')
dic = {
'time_now_chicago': now,
'time_now_chicago_short': now_short,
'report_created': created,
'report_created_short': created_short,
'bot': bot,
'bot_timezone': bot_timezone,
'source_id': id,
'exists': False,
'has_extended_profile': '',
'description': '',
'profile_use_background_image': '',
'time_zone': '',
'verified': '',
'followers_count': '',
'geo_enabled': '',
'lang': '',
'statuses_count': '',
'friends_count': '',
'favourites_count': '',
'account_created': '',
'account_created_short': '',
'location': '',
'default_profile': '',
'profile_image_url': '',
'screen_name': ''
}
with open(outfile, "a") as ffile:
counter = counter + 1
print(counter)
json.dump(unicode(dic), ffile, encoding='utf-8')
ffile.write("\n")
l = f.readline()
f.close()
