def insert_data(self, data): """ Will handle inserting data into the database """ try: db_session = DBSession() # Check if comic is in database, if so update else create try: comic = db_session.query(Comic).filter( Comic.comic_id == data.get('comic_id')).one() except NoResultFound: comic = Comic() comic.title = data.get('title') comic.alt = data.get('alt') comic.comic_id = data.get('comic_id') comic.source_file_location = data.get('source_file_location') comic.saved_file_location = data.get('saved_file_location') comic.posted_at = data.get('posted_at') comic.raw_json = data.get('raw_json') comic.time_collected = data.get('time_collected') comic.transcript = data.get('transcript') db_session.add(comic) db_session.commit() # self.track_stat('rows_added_to_db', rows_affected) except Exception: db_session.rollback() logger.exception("Error adding to db {data}".format(data=data))
def insert_data(self, data): """ Will handle inserting data into the database """ try: db_session = DBSession() # Check if comic is in database, if so update else create try: comic = db_session.query(Comic).filter( Comic.comic_id == data.get('comic_id')).one() except NoResultFound: comic = Comic() comic.title = data.get('title') comic.comic_id = data.get('id') comic.alt = data.get('alt') comic.file_path = data.get('file_path') comic.posted_at = data.get('posted_at') comic.time_collected = data.get('time_collected') db_session.add(comic) db_session.commit() except Exception: db_session.rollback() logger.exception("Error adding to db {data}".format(data=data))
def get_info(comic_num): URL = "https://www.explainxkcd.com/wiki/index.php/%s" % str(comic_num) soup = make_soup(URL) if soup == None: return "Error: comic %d not found" % comic_num result = Comic(comic_num) # get title result.og_title = get_title(comic_num) result.title = clean_text(result.og_title.split()) # get transcript try: transcript = soup.find("span", {"id": "Transcript"}) result.transcript = [] cur = transcript.parent while cur: if cur.name == 'dl': for dd in cur: result.transcript.append( str(dd).strip('<dd>').strip('</dd>')) elif cur.name == 'span': break cur = cur.nextSibling result.transcript = clean_text((" ".join(result.transcript)).split()) except: result.transcript = [] # get title text result.og_ttext = get_ttext(comic_num) result.title_text = clean_text(result.og_ttext.split()) # get explanation try: explanation = soup.find("span", {"id": "Explanation"}) result.explanation = [] cur = explanation.parent while cur: if cur.name == 'p': result.explanation.append(cur.text) elif cur.name == 'span': break cur = cur.nextSibling result.explanation = clean_text((" ".join(result.explanation)).split()) except: result.explanation = [] # get image URL result.img_url = get_img_url(comic_num) return result
def post(self): comics = self.request.body logging.warning(comics) comics = json.loads(comics) for comic in comics: newComic = Comic(publisher=comic.get('publisher'), title=comic.get('title'), booknum=comic.get('booknum'), writer=comic.get('writer'), artist=comic.get('artist'), misc=comic.get('misc')) newComic.put() self.response.out.write(newComic.key.urlsafe())
def test_add_comic(self): newComic = Comic(9021090, "Test Comic", 4, "Test Desc", "12345678", 88, 4.99, "http://www.utexas.edu/diploma.gif", 6, 7, 3) db.session.add(newComic) db.session.commit() query = db.session.query(Comic).filter_by(id="9021090").first() self.assertEqual(query.id, 9021090) self.assertEqual(query.title, "Test Comic") self.assertEqual(query.img, "http://www.utexas.edu/diploma.gif") self.assertEqual(query.num_creators, 6) self.assertEqual(query.num_characters, 7) self.assertEqual(query.num_events, 3) db.session.delete(newComic) db.session.commit()
def test_add_comic(self): with app.app_context(): newComic = Comic(id=9021090, title="Test Comic", issue_num=4, desc="Test Desc", upc="12345678", pg_ct=88, price=4.99, img="http://www.utexas.edu/diploma.gif", num_creators=6, num_characters=7, num_events=3) db.session.add(newComic) db.session.commit() query = db.session.query(Comic).filter_by(id="9021090").first() self.assertEqual(query.id, 9021090) self.assertEqual(query.title, "Test Comic") self.assertEqual(query.img, "http://www.utexas.edu/diploma.gif") self.assertEqual(query.num_creators, 6) self.assertEqual(query.num_characters, 7) self.assertEqual(query.num_events, 3) db.session.delete(newComic) db.session.commit()
def get_comics(self, publisher_id): rows = self.database.get_comics(publisher_id) result = [] for row in rows: result.append(Comic(row)) return result
def sample_comic(client): comic = Comic(title='Shade The Changing Girl', issues='12') db.session.add(comic) db.session.commit() return comic
def loadArcamax(): arcamax = [{ "meta": "ninechickweedlane", "name": "9 Chickweed Lane" }, { "meta": "agnes", "name": "Agnes" }, { "meta": "andycapp", "name": "Andy Capp" }, { "meta": "arcticcircle", "name": "Arctic Circle" }, { "meta": "bc", "name": "BC" }, { "meta": "babyblues", "name": "Baby Blues" }, { "meta": "beetlebailey", "name": "Beetle Bailey" }, { "meta": "blondie", "name": "Blondie" }, { "meta": "boondocks", "name": "Boondocks" }, { "meta": "brilliantmindofedisonlee", "name": "Brilliant Mind of Edison Lee" }, { "meta": "cathy", "name": "Cathy" }, { "meta": "daddyshome", "name": "Daddy's Home" }, { "meta": "dilbert", "name": "Dilbert" }, { "meta": "thedinetteset", "name": "Dinette Set" }, { "meta": "dogeatdoug", "name": "Dog Eat Doug" }, { "meta": "doonesbury", "name": "Doonesbury" }, { "meta": "dustin", "name": "Dustin" }, { "meta": "familycircus", "name": "Family Circus" }, { "meta": "garfield", "name": "Garfield" }, { "meta": "getfuzzy", "name": "Get Fuzzy" }, { "meta": "girlsandsports", "name": "Girls & Sports" }, { "meta": "hagarthehorrible", "name": "Hagar the Horrible" }, { "meta": "heathcliff", "name": "Heathcliff" }, { "meta": "hiandlois", "name": "Hi and Lois" }, { "meta": "humorcartoon", "name": "Jerry King Cartoons" }, { "meta": "luann", "name": "Luann" }, { "meta": "mallardfillmore", "name": "Mallard Fillmore" }, { "meta": "momma", "name": "Momma" }, { "meta": "mothergooseandgrimm", "name": "Mother Goose & Grimm" }, { "meta": "mutts", "name": "Mutts" }, { "meta": "nonsequitur", "name": "Non Sequitur" }, { "meta": "pearlsbeforeswine", "name": "Pearls Before Swine" }, { "meta": "pickles", "name": "Pickles" }, { "meta": "redandrover", "name": "Red and Rover" }, { "meta": "rhymeswithorange", "name": "Rhymes with Orange" }, { "meta": "rubes", "name": "Rubes" }, { "meta": "rugrats", "name": "Rugrats" }, { "meta": "speedbump", "name": "Speed Bump" }, { "meta": "wizardofid", "name": "Wizard of Id" }, { "meta": "zits", "name": "Zits" }] for y in arcamax: c = Comic(name=y['name'], author="", ctype=ARCAMAX_TYPE, meta=y['meta'], num_entries=0) c.put()
def loadXKCD(): c = Comic(name="XKCD", author="Randall Munroe", ctype=XKCD_TYPE, num_entries=0) c.put()
def loadComicsCom(): comicscom = [{ "name": "9 Chickweed Lane", "meta": "9_chickweed_lane" }, { "name": "Agnes", "meta": "agnes" }, { "name": "Alley Oop", "meta": "alley_oop" }, { "name": "Andy Capp", "meta": "andy_capp" }, { "name": "Arlo & Janis", "meta": "arlo&janis" }, { "name": "B.C.", "meta": "bc" }, { "name": "Ballard Street", "meta": "ballard_street" }, { "name": "Ben", "meta": "ben" }, { "name": "Betty", "meta": "betty" }, { "name": "Big Nate", "meta": "big_nate" }, { "name": "The Born Loser", "meta": "the_born_loser" }, { "name": "Brevity", "meta": "brevity" }, { "name": "The Buckets", "meta": "the_buckets" }, { "name": "Candorville", "meta": "candorville" }, { "name": "Cheap Thrills", "meta": "cheap_thrills" }, { "name": "Committed", "meta": "committed" }, { "name": "Cow & Boy", "meta": "cow&boy" }, { "name": "Dilbert", "meta": "dilbert" }, { "name": "The Dinette Set", "meta": "the_dinette_set" }, { "name": "Dog eat Doug", "meta": "dog_eat_doug" }, { "name": "Drabble", "meta": "drabble" }, { "name": "Family Tree", "meta": "family_tree" }, { "name": "Farcus", "meta": "farcus" }, { "name": "Fat Cats Classics", "meta": "fat_cats_classics" }, { "name": "Flight Deck", "meta": "flight_deck" }, { "name": "Flo & Friends", "meta": "flo&friends" }, { "name": "F Minus", "meta": "f_minus" }, { "name": "Frank & Ernest", "meta": "frank&ernest" }, { "name": "Frazz", "meta": "frazz" }, { "name": "Free Range", "meta": "free_range" }, { "name": "Geech Classics", "meta": "geech_classics" }, { "name": "Get Fuzzy", "meta": "get_fuzzy" }, { "name": "Girls & Sports", "meta": "girls&sports" }, { "name": "Graffiti", "meta": "graffiti" }, { "name": "Grand Avenue", "meta": "grand_avenue" }, { "name": "The Grizzwells", "meta": "the_grizzwells" }, { "name": "Heathcliff", "meta": "heathcliff" }, { "name": "Herb and Jamaal", "meta": "herb_and_jamaal" }, { "name": "Herman", "meta": "herman" }, { "name": "The Humble Stumble", "meta": "the_humble_stumble" }, { "name": "Jump Start", "meta": "jump_start" }, { "name": "Kit 'N' Carlyle", "meta": "kit_n_carlyle" }, { "name": "The Knight Life", "meta": "the_knight_life" }, { "name": "Liberty Meadows", "meta": "liberty_meadows" }, { "name": "Li'l Abner Classics", "meta": "lil_abner_classics" }, { "name": "Lola", "meta": "lola" }, { "name": "Luann", "meta": "luann" }, { "name": "Marmaduke", "meta": "marmaduke" }, { "name": "Meg! Classics", "meta": "meg_classics" }, { "name": "The Meaning of Lila", "meta": "the_meaning_of_lila" }, { "name": "Minimum Security", "meta": "minimum_security" }, { "name": "Moderately Confused", "meta": "moderately_confused" }, { "name": "Momma", "meta": "momma" }, { "name": "Monty", "meta": "monty" }, { "name": "Motley Classics", "meta": "motley_classics" }, { "name": "Nancy", "meta": "nancy" }, { "name": "Natural Selection", "meta": "natural_selection" }, { "name": "Off The Mark", "meta": "off_the_mark" }, { "name": "On a Claire Day", "meta": "on_a_claire_day" }, { "name": "One Big Happy Classics", "meta": "one_big_happy_classics" }, { "name": "The Other Coast", "meta": "the_other_coast" }, { "name": "Over the Hedge", "meta": "over_the_hedge" }, { "name": "PC and Pixel", "meta": "pc_and_pixel" }, { "name": "Pearls Before Swine", "meta": "pearls_before_swine" }, { "name": "Pickles", "meta": "pickles" }, { "name": "Raising Duncan Classics", "meta": "raising_duncan_classics" }, { "name": "Reality Check", "meta": "reality_check" }, { "name": "Ripley's Believe It or Not!", "meta": "ripleys_believe_it_or_not" }, { "name": "Rose Is Rose", "meta": "rose_is_rose" }, { "name": "Rubes", "meta": "rubes" }, { "name": "Rudy Park", "meta": "rudy_park" }, { "name": "Shirley and Son Classics", "meta": "shirley_and_son_classics" }, { "name": "Soup To Nutz", "meta": "soup_to_nutz" }, { "name": "Speed Bump", "meta": "speed_bump" }, { "name": "Spot The Frog", "meta": "spot_the_frog" }, { "name": "Strange Brew", "meta": "strange_brew" }, { "name": "The Sunshine Club", "meta": "the_sunshine_club" }, { "name": "Tarzan Classics", "meta": "tarzan_classics" }, { "name": "That's Life", "meta": "thats_life" }, { "name": "Watch Your Head", "meta": "watch_your_head" }, { "name": "Wizard of Id", "meta": "wizard_of_id" }, { "name": "Working Daze", "meta": "working_daze" }, { "name": "Working It Out", "meta": "working_it_out" }, { "name": "Zack Hill", "meta": "zack_hill" }, { "name": "Ferd'nand", "meta": "ferdnand" }, { "name": "Rip Haywire", "meta": "rip_haywire" }, { "name": "The Barn", "meta": "the_barn" }, { "name": "Home and Away", "meta": "home_and_away" }, { "name": "Little Dog Lost", "meta": "little_dog_lost" }, { "name": "Daddy's Home", "meta": "daddys_home" }, { "name": "Scary Gary", "meta": "scary_gary" }, { "name": "Nest Heads", "meta": "nest_heads" }, { "name": "Unstrange Phenomena", "meta": "unstrange_phenomena" }, { "name": "Prickly City", "meta": "prickly_city" }, { "name": "Fort Knox", "meta": "fort_knox" }, { "name": "Cafe Con Leche", "meta": "cafe_con_leche" }, { "name": "Animal Crackers", "meta": "animal_crackers" }, { "name": "Bliss", "meta": "bliss" }, { "name": "Bottom Liners", "meta": "bottom_liners" }, { "name": "Bound and Gagged", "meta": "bound_and_gagged" }, { "name": "Brewster Rockit", "meta": "brewster_rockit" }, { "name": "Broom-Hilda", "meta": "broom-hilda" }, { "name": "Dick Tracy", "meta": "dick_tracy" }, { "name": "Gasoline Alley", "meta": "gasoline_alley" }, { "name": "Gil Thorp", "meta": "gil_thorp" }, { "name": "Loose Parts", "meta": "loose_parts" }, { "name": "Love Is...", "meta": "love_is" }, { "name": "The Middletons", "meta": "the_middeltons" }, { "name": "Pluggers", "meta": "pluggers" }, { "name": "Sylvia", "meta": "sylvia" }, { "name": "9 to 5", "meta": "9_to_5" }, { "name": "Chuckle Bros", "meta": "chuckle_bros" }, { "name": "Barney & Clyde", "meta": "barney&clyde" }, { "name": "Today's Dogg", "meta": "todays_dogg" }, { "name": "Freshly Squeezed", "meta": "freshly_squeezed" }, { "name": "Frazz Holiday", "meta": "frazz_holiday" }, { "name": "Dude and Dude", "meta": "dude_and_dude" }, { "name": "Reply All", "meta": "reply_all" }] for y in comicscom: c = Comic(name=y['name'], author="", ctype=COMIC_TYPE, meta=y['meta'], num_entries=0) c.put()
def loadPhd(): c = Comic(name="Piled Higher & Deeper", author="Jorge Cham", ctype=PHD_TYPE, num_entries=0) c.put()
def loadYahoo(): yahoo = [{ "meta": "two-cows-chicken", "name": u"2 Cows and a Chicken" }, { "meta": "9-chickweed-lane", "name": u"9 Chickweed Lane" }, { "meta": "adam-at-home", "name": u"Adam @ Home" }, { "meta": "alley-oop", "name": u"Alley Oop" }, { "meta": "andy-capp", "name": u"Andy Capp" }, { "meta": "argyle-sweater", "name": u"The Argyle Sweater" }, { "meta": "arlo-and-janis", "name": u"Arlo and Janis" }, { "meta": "bc", "name": u"B.C." }, { "meta": "baldo", "name": u"Baldo" }, { "meta": "ballard-street", "name": u"Ballard Street" }, { "meta": "basic-instructions", "name": u"Basic Instructions" }, { "meta": "betty", "name": u"Betty" }, { "meta": "big-nate", "name": u"Big Nate" }, { "meta": "big-picture", "name": u"The Big Picture" }, { "meta": "big-top", "name": u"Big Top" }, { "meta": "bob-the-squirrel", "name": u"Bob the Squirrel" }, { "meta": "boondocks", "name": u"Boondocks" }, { "meta": "born-loser", "name": u"Born Loser" }, { "meta": "brevity", "name": u"Brevity" }, { "meta": "brewster-rockit", "name": u"Brewster Rockit" }, { "meta": "the-buckets", "name": u"The Buckets" }, { "meta": "classic-bloom-county", "name": u"Classic Bloom County" }, { "meta": "candorville", "name": u"Candorville" }, { "meta": "cathy", "name": u"Cathy" }, { "meta": "cleats", "name": u"Cleats" }, { "meta": "close-to-home", "name": u"Close to Home" }, { "meta": "committed", "name": u"Committed" }, { "meta": "cornered", "name": u"Cornered" }, { "meta": "cow-and-boy", "name": u"Cow and Boy" }, { "meta": "cul-de-sac", "name": u"Cul-de-Sac" }, { "meta": "dark-side-horse", "name": u"Dark Side of the Horse" }, { "meta": "dilbert", "name": u"Dilbert" }, { "meta": "the-dinette-set", "name": u"The Dinette Set" }, { "meta": "doonesbury", "name": u"Doonesbury" }, { "meta": "doozies", "name": u"The Doozies" }, { "meta": "drabble", "name": u"Drabble" }, { "meta": "dude-and-dude", "name": u"Dude and Dude" }, { "meta": "the-duplex", "name": u"The Duplex" }, { "meta": "elderberries", "name": u"The Elderberries" }, { "meta": "family-tree", "name": u"Family Tree" }, { "meta": "farcus", "name": u"Farcus" }, { "meta": "fat-cats-classics", "name": u"Fat Cats Classics" }, { "meta": "ferdnand", "name": u"Ferd'nand" }, { "meta": "the-flying-mccoys", "name": u"The Flying McCoys" }, { "meta": "f-minus", "name": u"F Minus" }, { "meta": "for-better-or-for-worse", "name": u"For Better or For Worse" }, { "meta": "foxtrot", "name": u"FoxTrot" }, { "meta": "foxtrot-classics", "name": u"FoxTrot Classics" }, { "meta": "frank-and-ernest", "name": u"Frank and Ernest" }, { "meta": "frazz", "name": u"Frazz" }, { "meta": "fred-basset", "name": u"Fred Basset" }, { "meta": "freshly-squeezed", "name": u"Freshly Squeezed" }, { "meta": "fusco-brothers", "name": u"The Fusco Brothers" }, { "meta": "garfield", "name": u"Garfield" }, { "meta": "geech-classics", "name": u"Geech Classics" }, { "meta": "get-fuzzy", "name": u"Get Fuzzy" }, { "meta": "graffiti", "name": u"Graffiti" }, { "meta": "grand-avenue", "name": u"Grand Avenue" }, { "meta": "the-grizzwells", "name": u"The Grizzwells" }, { "meta": "heart-of-the-city", "name": u"Heart of the City" }, { "meta": "heathcliff", "name": u"Heathcliff" }, { "meta": "housebroken", "name": u"Housebroken" }, { "meta": "herman", "name": u"Herman" }, { "meta": "ink-pen", "name": u"Ink Pen" }, { "meta": "in-the-bleachers", "name": u"In the Bleachers" }, { "meta": "in-the-sticks", "name": u"In The Sticks" }, { "meta": "janes-world", "name": u"Jane's World" }, { "meta": "jumpstart", "name": u"Jumpstart" }, { "meta": "kit-n-carlyle", "name": u"Kit 'n' Carlyle" }, { "meta": "knight-life", "name": u"The Knight Life" }, { "meta": "la-cucaracha", "name": u"La Cucaracha" }, { "meta": "lio", "name": u"Lio" }, { "meta": "lola", "name": u"Lola" }, { "meta": "luann", "name": u"Luann" }, { "meta": "marmaduke", "name": u"Marmaduke" }, { "meta": "meg-classics", "name": u"Meg! Classics" }, { "meta": "minimum-security", "name": u"Minimum Security" }, { "meta": "moderately-confused", "name": u"Moderately Confused" }, { "meta": "momma", "name": u"Momma" }, { "meta": "monty", "name": u"Monty" }, { "meta": "motley-classics", "name": u"Motley Classics" }, { "meta": "nancy", "name": u"Nancy" }, { "meta": "new-adventures-queen-victoria", "name": u"New Adventures of Queen Victoria" }, { "meta": "non-sequitur", "name": u"Non Sequitur" }, { "meta": "off-the-mark", "name": u"Off the Mark" }, { "meta": "one-big-happy", "name": u"One Big Happy" }, { "meta": "overboard", "name": u"Overboard" }, { "meta": "over-the-hedge", "name": u"Over the Hedge" }, { "meta": "pc-and-pixel", "name": u"PC and Pixel" }, { "meta": "peanuts", "name": u"Peanuts" }, { "meta": "pearls-before-swine", "name": u"Pearls Before Swine" }, { "meta": "pooch-cafe", "name": u"Pooch Café" }, { "meta": "preteena", "name": u"PreTeena" }, { "meta": "prickly-city", "name": u"Prickly City" }, { "meta": "reality-check", "name": u"Reality Check" }, { "meta": "real-life-adventures", "name": u"Real Life Adventures" }, { "meta": "red-rover", "name": u"Red & Rover" }, { "meta": "rip-haywire", "name": u"Rip Haywire" }, { "meta": "ripleys-believe-it-or-not", "name": u"Ripley's Believe it or Not" }, { "meta": "rose-is-rose", "name": u"Rose is Rose" }, { "meta": "rubes", "name": u"Rubes" }, { "meta": "rudy-park", "name": u"Rudy Park" }, { "meta": "shirley-and-son-classics", "name": u"Shirley and Son Classics" }, { "meta": "shoe", "name": u"Shoe" }, { "meta": "skin-horse", "name": u"Skin Horse" }, { "meta": "slowpoke", "name": u"Slowpoke" }, { "meta": "soup-to-nutz", "name": u"Soup to Nutz" }, { "meta": "speed-bump", "name": u"Speed Bump" }, { "meta": "stone-soup", "name": u"Stone Soup" }, { "meta": "strange-brew", "name": u"Strange Brew" }, { "meta": "the-sunshine-club", "name": u"The Sunshine Club" }, { "meta": "tank-mcnamara", "name": u"Tank McNamara" }, { "meta": "thatababy", "name": u"Thatababy" }, { "meta": "thats-life", "name": u"That'sLife" }, { "meta": "tiny-sepuku", "name": u"Tiny Sepuku" }, { "meta": "toby", "name": u"Toby" }, { "meta": "tom-dancing-bug", "name": u"Tom the Dancing Bug" }, { "meta": "too-much-coffee-man", "name": u"Too Much Coffee Man" }, { "meta": "unstrange-phenomena", "name": u"Unstrange Phenomena" }, { "meta": "wizard-of-id", "name": u"Wizard of Id" }, { "meta": "working-daze", "name": u"Working Daze" }, { "meta": "wt-duck", "name": u"W.T. Duck" }, { "meta": "ziggy", "name": u"Ziggy" }] for y in yahoo: c = Comic(name=y['name'], author="", ctype=YAHOO_TYPE, meta=y['meta'], num_entries=0) c.put()
from storage import ComicManager from models import Comic cm_lite = ComicManager('sqlite:///foxtrot.db') cm_mysql = ComicManager( 'mysql://*****:*****@gregoria.mysql.pythonanywhere-services.com/gregoria$foxtrot?charset=utf8' ) with cm_lite.session_scope() as s_lite: comics = s_lite.query(Comic) with cm_mysql.session_scope() as s_my: for comic in comics: new_comic = Comic(date=comic.date, transcript=comic.transcript) s_my.add(new_comic)
try: event, values = window.read(timeout=100) if event in [sg.WIN_CLOSED, 'Exit']: break if event == '-SELECTALL-': if not values['-SELECTALL-']: window['-CHAPTERLIST-'].set_value([]) elif comic is not None: chapters_names = [chapter.title for chapter in comic.chapters] window['-CHAPTERLIST-'].set_value(chapters_names) if event == 'Fetch': url = values['-COMICURL-'] if len(url) <= 0 or 'http' not in url: sg.Popup('Please type in URL') continue comic = Comic(url) comic.set_title() window['-COMICNAME-'].update(comic.title) comic.set_chapters() window['-CHAPTERLIST-'].update( [chapter.title for chapter in comic.chapters]) if event == 'Download': if comic is None: sg.Popup('Please type in url and fetch') if len(values['-SAVETO-']) <= 0: sg.Popup('Please choose folder to save') popup_text = 'Do you wish to start downloading {}?\n'.format( comic.title) for chapter_name in values['-CHAPTERLIST-']: popup_text = popup_text + '- {}\n'.format(chapter_name) ok = sg.PopupYesNo(popup_text)
def main(): #fcharacters = open('comic_characters.txt', 'a') #fcreators = open('comic_creators.txt', 'a') #fevents = open('comic_events.txt', 'a') marvel = MarvelRequest() """ json.loads(String) takes in json formatted string, and outputs data according to the conversion table at json library website """ index = 0 for offset in range(36080, 41000, 20): response = marvel.request("comics", offset) # No trailing slash allowed here print(response.status_code) assert response.status_code == 200 comic = json.loads(response.text) idNum = 0 title = "" issue = "" desc = "" upc = "" pg_ct = 0 comic_price = "" path = "" numCreators = "" numChars = "" numEvents = "" for comic_meta_keys, comic_meta_data in comic['data'].items(): # comic_meta_keys: offset, limit, total, count, results[] from # Marvel JSON structure if comic_meta_keys == 'results': for comic in comic_meta_data: if comic['id'] != "": for comic_attribute_keys, comic_attribute in comic.items(): # now stepping through title, description, thumbnail, etc. if comic_attribute_keys == 'id': idNum = int(comic_attribute) # idNum = idNum.encode('utf-8') elif comic_attribute_keys == 'title': title = comic_attribute title = title.encode('utf-8') #print('Title: ' + title) elif comic_attribute_keys == 'issueNumber': issue = int(comic_attribute) elif comic_attribute_keys == 'variantDescription': desc = comic_attribute elif comic_attribute_keys == 'description': if (comic_attribute != None) and (comic_attribute != ""): """ Error arose when using str(description) and transferring output to text file: You must not use str(...) to strip away unicode symbols that often appear in Marvel descriptions! """ desc = comic_attribute #desc = desc.encode('utf-8') #print('Description: ' + desc) elif comic_attribute_keys == 'upc': #print("Start Year: " + str(comic_attribute)) upc = str(comic_attribute) elif comic_attribute_keys == 'pageCount': #print("End Year: " + str(comic_attribute)) pg_ct = int(comic_attribute) elif comic_attribute_keys == 'prices': d = comic_attribute[0] comic_price = float(d.get('price')) #print(comic_price) elif comic_attribute_keys == 'thumbnail': path = str(comic_attribute['path']) temp = path.split('/') for v in temp : if v == 'image_not_available': path = None if path != None: path = str(path) + '.' + str(comic_attribute['extension']) # print (path) #if comic_attribute_keys == 'id': # idNum = int(comic_attribute) elif comic_attribute_keys == 'creators': # print("Comics in comic: " + str(comic_attribute['available'])) numCreators = int(comic_attribute['available']) #creator_ids = [idNum] #for creator_uri in comic_attribute['items']: # resource_path = creator_uri['resourceURI'].split('/') # creator_ids.append(int(resource_path[-1])) # fcreators.write(str(creator_ids) + '\n') elif comic_attribute_keys == 'characters': # print("Characters in comic: " + str(comic_attribute['available'])) numChars = int(comic_attribute['available']) #character_ids = [comic['id']] #for character in comic_attribute['items']: # resource_path = character['resourceURI'].split('/') # character_ids.append(int(resource_path[-1])) #fcharacters.write(str(character_ids) + '\n') elif comic_attribute_keys == 'events': numEvents = int(comic_attribute['available']) #event_ids = [idNum] #for event in comic_attribute['items']: # resource_path = event['resourceURI'].split('/') # event_ids.append(int(resource_path[-1])) # fevents.write(str(event_ids) + '\n') newEntry = Comic(idNum, title, issue, desc, upc, pg_ct, comic_price, path, numCreators, numChars, numEvents) db.session.merge(newEntry) db.session.commit() index += 1 print("processed comic " + str(index))
def post(self): comic = Comic(name=self.request.get('inputName'), description=self.request.get('inputDescription'), cover=self.request.get('inputURL')) comic.put() return webapp2.redirect('/')