def setUpClass(cls):
df = pd.read_csv(io.StringIO("publish_date,headline_text,publish_yearmonth,publish_month\n20150409,rural sa rural reporter the tale of two orchards,201504,04\n20111206,roar get ulsan in champions league draw,201112,12\n20101201,130m annual cost to run desal plant,201012,12\n20040802,farmers worried about wto agreement loopholes,200408,08\n20170808,same sex marriage plebiscite attempt expected to be blocked,201708,08\n20130621,executives spend a night on the streets to experience homelessn,201306,06\n20070613,nsw govt signs pollution reduction agreement with,200706,06\n20060209,nt doctors show support for abortion drug,200602,02\n20130718,crash driver sought by police,201307,07\n20061119,howard disputes blairs iraq comments,200611,11\n20070725,german reporter released in afghanistan,200707,07\n20120224,hammer heal to coach kings,201202,02\n20090428,written apology over holocaust denial,200904,04\n20141024,unions hand tasmanian government alternative savings plan,201410,10\n20061118,shark gets some pride back,200611,11\n20130206,older watson concerned for jobe bombers,201302,02\n20140430,forum to showcase mid west mining developments,201404,04\n20140429,former wa treasurer buswell admits to driving offences,201404,04\n20070621,weather to determine sports fields opening,200706,06\n20140803,travel blamed for increasing rate of hiv in wa,201408,08\n20050715,stuey takes aim at green jersey,200507,07\n20061219,public urged to help combat fruit fly threat,200612,12\n20040302,robben chooses chelsea over united,200403,03\n20030820,jury to continue deliberations in hanson fraud,200308,08\n20030323,baghdads military facilities targeted in latest,200303,03\n20140417,an india holds biggest day of voting,201404,04\n20050102,car bomb attack kills 18 iraqi national guards,200501,01\n20080818,citation boosts vietnam veterans day significance,200808,08\n20131111,wenceslas magun speaks to pacific beat,201311,11\n20130325,an vanuatu gets new pm,201303,03\n20160423,woman killed in crash with stobie pole,201604,04\n20091006,message spread that attacks not tolerated brumby,200910,10\n20040707,iraq adopts new security laws,200407,07\n20030916,poland gets record case of the blues,200309,09\n20040406,jordan sentences eight to death over diplomat,200404,04\n20101022,arnold to relish cox plate pressure,201010,10\n20130610,lack of data creates concern over true extent of medical errors,201306,06\n20060317,labor warns on minority government,200603,03\n20100808,labor to ban truants from playing sport,201008,08\n20071210,sharks spotted in esperance port,200712,12\n20041224,aust troops to celebrate christmas in iraq,200412,12\n20090819,jail term for rsl theft,200908,08\n20070408,closer am1nodisplay,200704,04\n20161114,nt man jailed for crimes against children,201611,11\n20051003,union warns ir changes threaten australian way of,200510,10\n20041007,afghan children lose high court battle against,200410,10\n20130506,parkinsons test sought,201305,05\n20110329,police accused of not probing brutality claim,201103,03\n20090828,cairns trip ends in top end lsd bust,200908,08\n20120816,coroner criticises ambulance 'ramping',201208,08\n20130121,new recruits for womens cycling team,201301,01\n20050203,uni to hold tropical science precinct talks,200502,02\n20041110,jetstar asia prepares for launch,200411,11\n20090326,mccreadie granted immunity,200903,03\n20170821,one killed in france after car crashes into bus shelters,201708,08\n20081031,gambhir handed one test ban,200810,10\n20150527,school communities unsettled about prospect of school closures,201505,05\n20050829,man accused of ramming car with children inside,200508,08\n20130821,van park owner pursues legal options over free,201308,08\n20060406,national network to track pseudoephedrine sales,200604,04\n20040708,big sports complex planned near maitland,200407,07\n20100714,ex afl player paid nearly 80k to conman,201007,07\n20120711,victory retain milligans services,201207,07\n20080221,bad weather delays dalrymple bay coal terminal,200802,02\n20151021,govt department tests scales get what paid for,201510,10\n20090208,battered jets sign italian striker vignaroli,200902,02\n20130205,capital hill monday 4 february 2013,201302,02\n20161013,medicinal cannabis register considered tasmania,201610,10\n20041202,underwood sworn in as chief justice,200412,12\n20110701,rta heeds call for pedestrian safety upgrade,201107,07\n20120723,miners say cost of business too high,201207,07\n20090103,funding secures more aerial shark patrols,200901,01\n20170603,were australias first people nomadic,201706,06\n20031019,tributes pour in for spanish writer montalban,200310,10\n20080301,interview ricky ponting,200803,03\n20100831,forlan at the double for atletico,201008,08\n20060907,lawyers say vizards silence is unfair to hilliard,200609,09\n20060524,shoulder troubles for roddick ahead of french,200605,05\n20080809,tennis form guide mens singles,200808,08\n20171206,family of betty dixon still asking questions as cold case ends,201712,12\n20080715,fed court overturns annoying ban,200807,07\n20120131,rare earth industry developing rapidly,201201,01\n20131117,tremlett prior set to start for england,201311,11\n20121114,eltons latest book explores brothers relationship,201211,11\n20070316,evans a man of honesty and integrity,200703,03\n20040908,financial lobby criticises labor tax package,200409,09\n20030604,health service urged to review gp anaesthetist,200306,06\n20030410,restrictions for melbourne as water cost rises,200304,04\n20161022,pamela anderson speaks out about pornographys numbing effects,201610,10\n20120804,fire warning,201208,08\n20110329,paramedic gives evidence at road crash murder trial,201103,03\n20160711,response to labor mp call to ban fracking in south west,201607,07\n20111007,health razor gang disbands early,201110,10\n20141023,acid attacks on women spark protests in iran,201410,10\n20100401,mp airs fears for forestry jobs,201004,04\n20121124,interview rianna ponting,201211,11\n20120820,tony burke talks with four corners,201208,08\n20100815,20 million affected by pakistan floods,201008,08\n20091222,china planning to execute briton next week,200912,12\n20100819,woman granted bail over torso in bush find,201008,08\n20091103,christmas island locals forgotten in asylum debate,200911,11\n20071027,eden monaro headed for labor poll,200710,10\n20121027,alleged hijackers flown to sri lanka to face charges,201210,10\n20160320,powerlifting: watch a benchpress; a deadlift and a,201603,03\n20130913,new york jets' mark sanchez facing season ending shoulder sur,201309,09\n20120324,we have to put bligh legacy behind us,201203,03\n20050524,budget sees return of investment properties tax,200505,05\n20101117,germany increases security amid terrorist threat,201011,11\n20150713,newcastle man in coma after drunken argument,201507,07\n20140812,titans need help in afl battle,201408,08\n20170119,vegemite back in australian hands,201701,01\n20070508,utai out cutler in for dogs,200705,05\n20160818,artists opens up world of picture book illustrations,201608,08\n20150731,north queensland ports urge ports bill fine tuning,201507,07\n20060623,wimmera sheep sales increase,200606,06\n20120105,opposition queries extra senior bureaucrats,201201,01\n20120514,hume result,201205,05\n20070909,victorians going green,200709,09\n20121113,broken hill baby birds back in their nests,201211,11\n20111023,drunk driving police,201110,10\n20070806,four arrested over safe breaks,200708,08\n20131214,sri lanka retain twenty20 number one ranking,201312,12\n20061122,sydney tourism snubs regional areas,200611,11\n20070512,curbishley confident of players resolve,200705,05\n20050924,ten killed in gaza hamas rally blast,200509,09\n20080804,police dig for baby 12 years on,200808,08\n20090602,centenary show for gin gin,200906,06\n20090426,g20 ministers still cautious on global economy swan,200904,04\n20080918,david kidman from ferrier hodgson talks about the,200809,09\n20091101,beauty with a twist,200911,11\n20091203,henderson talks up brave 2030 plan,200912,12\n20070913,power in no rush to decide political future,200709,09\n20091209,swine flu far milder than feared,200912,12\n20091216,us house of reps honours miles davis album,200912,12\n20160816,two dead in crash on eyre highway near balladonia,201608,08\n20091022,worms linked to coeliac relief,200910,10\n20140401,wafarmers urges growers to decrease debt,201404,04\n20121115,fmg diversifies into oil and gas,201211,11\n20040121,leaders may need to resolve trade talks,200401,01\n20081207,tasmanians urged to spend within their means,200812,12\n20140822,sa police join search for missing warrnambool man,201408,08\n20051219,company fined after explosions injured workers,200512,12\n20081013,thai queen to attend protesters funeral,200810,10\n20111124,global stocks close,201111,11\n20051221,aquaculture group upset with course axing,200512,12\n20121224,somali troops end hostages' three year ordeal,201212,12\n20090804,bligh vows to refer email row to cmc,200908,08\n20100714,appointed to healths top job,201007,07\n20100128,remote schools low on my school site,201001,01\n20140505,festival visitors get taste for regions produce,201405,05\n20030413,canegrowers push for ethanol mix in all petrol,200304,04\n20110409,clarke ton helps aussies to victory,201104,04\n20151207,police seek witnesses to fatal tintinara road crash,201512,12\n20041013,tax relief tipped for wa home buyers,200410,10\n20050312,bulls charge towards home final,200503,03\n20151125,three men dead in perth workplace accidents,201511,11\n20160516,federal government considers assistance package dairy farmers,201605,05\n20130523,minister jeanette powell outlines strategy for victoria's abo,201305,05\n20140919,jackson primary school censorship,201409,09\n20090909,russians behind cyber crime says afp,200909,09\n20030709,indias congress considers coalition to oust bjp,200307,07\n20050425,council plans memorial to grassby,200504,04\n20090810,slovak mine blast traps 19 miners,200908,08\n20121123,some tourism operators say no to schoolies,201211,11\n20150507,australian farming families the feature of a new,201505,05\n20120322,young roos,201203,03\n20101206,katich has scans on achilles injury,201012,12\n20070627,pricey sydney tops census again,200706,06\n20060319,opals enjoy another big win,200603,03\n20160318,albany residents to be quizzed over muttonbird reserve,201603,03\n20150902,china fta senator colbeck trade,201509,09\n20160609,greyhound racing nsw charges 179 trainers owners,201606,06\n20060220,internet smss blamed for big crowd at party,200602,02\n20031203,renison mine to remain closed,200312,12\n20151215,newcastle giving tree finished for 2015,201512,12\n20070707,afp release five doctors after questioning,200707,07\n20121130,an bangladesh inspections,201211,11\n20121008,man quizzed over high speed chase,201210,10\n20080409,lennon under fire over kons resignation,200804,04\n20130510,compo concerns,201305,05\n20150730,police plead for clues to tenterden road crash,201507,07\n20081014,an open and shut case for nw road,200810,10\n20100511,scott daughters settle estate fight,201005,05\n20080523,suitability of hensons images depends on context,200805,05\n20060622,aged care group restructures decision making,200606,06\n20150204,nff wants banks to pass on interest rate cut to farmers,201502,02\n20041118,govts urged to act on commuter train service,200411,11\n20030323,worldwide protests demand peace,200303,03\n20040601,gillespie talks up worth of zimbabwe series,200406,06\n20050506,tribunal cracks down on video evidence,200505,05\n20151021,police make arrest missing mother linda sidon gold coast,201510,10\n20121012,scientists uncover mystery of ball lightning,201210,10\n20140430,encouraging girls in engineering jpbs,201404,04\n20160816,woman charged over assault of victorian labor mp jane garrett,201608,08\n20140224,cattle saleyards canteen ladies,201402,02\n20080726,final showdown looms for tour,200807,07\n20111229,pesce a rising tide of chaos,201112,12\n20040426,former us ambassador doubts iraq wmd focus,200404,04\n20080603,evicted aborigines finish training in sydney,200806,06\n20070412,cadets to attend sandakan dawn service,200704,04\n20100425,red shirts discarded ahead of crackdown,201004,04\n20070625,four to appear in court over coolgardie burglary,200706,06\n20140812,nrn graincorp ceo,201408,08\n20101230,interview michael clarke,201012,12\n20110506,workers to mine tafe for education needs,201105,05\n20130912,wafl player has bail varied to play,201309,09\n20120809,simpson elected murray irrigation shareholder,201208,08\n20121206,ice blamed for crime spike,201212,12\n20080622,opec divided on saudi summit and production boost,200806,06\n20050513,heroin bust in adelaide,200505,05\n20051004,nrma highlights need for pacific highway attention,200510,10\n20110706,public quizzed about closed inlet,201107,07\n20150225,herbicide resistance peter newman,201502,02\n20050216,push for second kakadu uranium mine,200502,02\n20040314,murali set to join warne in 500 wicket club,200403,03\n20131104,soil carbon climate change,201311,11\n20100208,the wwfs paul gamblin says a report should put,201002,02\n20040922,indonesian presidential hopeful plans peace in aceh,200409,09\n20170405,bushfire emergency downgraded near esperance in wa,201704,04\n20120724,injured sea birds washing up inland,201207,07\n20160729,donald trump v hillary clinton star power of the conventions,201607,07\n20120522,impact of bomb blasts on the brain,201205,05\n20140811,israel palestine agree to 72 hour cease fire in gaza,201408,08\n20130610,14yos accused of armed robbery,201306,06\n20051114,mp says tafe fees soaring,200511,11\n20050419,woolworths sales up more than 14pc,200504,04\n20080907,peter leek breaks butterfly world record,200809,09\n20080426,jones trickett set new world records,200804,04\n20041224,karzai removes warlords from afghan cabinet,200412,12\n20120329,no confidence showdown looming,201203,03\n20110114,brazil floods mudslides kill hundreds,201101,01\n20160918,hospital parking fees petition gains support on change org,201609,09\n20140716,china gdp growth hits expectations,201407,07\n20071206,pasha findings prompt port review,200712,12\n20080627,pigeons smuggle drugs phones into rio prison,200806,06\n20071228,plucky india fights back in melbourne,200712,12\n20150419,thousands in germany protest against ttip europe us trade deal,201504,04\n20100112,rain sets up new crop for cane farmers,201001,01\n20110110,peter andre named hardest working singer,201101,01\n20120830,search becomes rescue as asylum boat found,201208,08\n20050715,manslaughter charge dropped in bondage case,200507,07\n20120822,laurie daley interview,201208,08\n20030601,williams silent on sydney ji unit claim,200306,06\n20060226,govt offers to buy back sydney harbour fishing,200602,02\n20061115,reward offered to catch roo shooter,200611,11\n20121128,report suggests turnaround for struggling boxed,201211,11\n20081024,november execution for bali bombers,200810,10\n20040513,ethnic sounds unite eurovision,200405,05\n20111128,murray darling authority chairman craig knowles,201111,11\n20160122,brisbane artist helps fans pay tribute to idols through nail art,201601,01\n20120821,australia too complacent,201208,08\n20070829,rudd pressures howard to pick election date,200708,08\n20171203,cooper cronk goes out on top announcing retirement from rep,201712,12\n20140212,oz shares surge after ceo announces departure,201402,02\n20060630,council happy to receive community funds for,200606,06\n20131113,lifeline helping miners prevent suicide,201311,11\n20100701,authorities fear grass fires deliberately lit,201007,07\n20040827,family hires security guard for protection,200408,08\n20110315,contempt of court charge against paper dropped,201103,03\n20030416,full text 13 point plan for iraq,200304,04\n20090704,nrl interview neil henry,200907,07\n20120306,sa courts,201203,03\n20060119,australia west indies postpone 2007 test series,200601,01\n20140603,bosnia finalises cup squad,201406,06\n20121127,victorian government backs down on scrapping fruit,201211,11\n20050131,perth kalgoorlie line set to reopen on weekend,200501,01\n20150428,chile volcano calbuco economy 600 million tourism eruption,201504,04\n20130313,grain prices rabobank,201303,03\n20140415,fia upholds ricciardo disqualification,201404,04\n20100425,pies embarrass dons on big stage,201004,04\n20120213,shining path leader captured,201202,02\n20160715,rescue plane goes down in goldfields hunt for missing man,201607,07\n20110901,storm wont appeal blairs ban,201109,09\n20131108,today tonight twist in gittany trial,201311,11\n20070413,tour boat profits blown away,200704,04\n20170921,farmers open the farm gate to combat carrot glut,201709,09\n20130507,qdo resignation,201305,05\n20060531,australian teams join quake aid efforts,200605,05\n20110705,bartos the public service numbers game,201107,07\n20060705,patient no shows end specialist medical service,200607,07\n20150804,multi million dollar northern farming system project,201508,08\n20171229,china foreign ministry denies claims its still,201712,12\n20110807,masterchef winner,201108,08\n20161006,for better or worse: four corners,201610,10\n20070308,rsl investigates veterans home care service,200703,03\n20090212,keane at the double for ireland,200902,02\n20080102,pakistan issues photos of bhutto death offers,200801,01\n20121113,pair charged following police shooting,201211,11\n20040304,hope for business chamber turnaround,200403,03\n20050226,cabinet to consider nightclub lock out plan,200502,02\n20061220,illawarra schools do well in hsc,200612,12\n20121112,data reveals strong regional rental markets,201211,11\n20060629,teen found safe after missing in bush for three,200606,06\n20060110,star studded field confirmed for johnnie walker,200601,01\n20120113,abc sport,201201,01\n20140702,trade balance slumps to near 2 billion deficit on fall in iron,201407,07\n20090928,star to be born again,200909,09\n20100712,experts warn against growing diabetes threat,201007,07\n20031212,rampaging roy wins cultural recognition,200312,12\n20081221,chinese warships to join anti piracy force,200812,12\n20040603,mayor highlights hidden amalgamation costs,200406,06\n20091013,locals threaten to block kokoda over crash compo,200910,10\n20081211,connex told to fix industrial dispute,200812,12\n20141204,ronja huon aquaculture salmon,201412,12\n20161102,private investor interest in henty pub,201611,11\n20100324,councils face off over oakajee,201003,03\n20160407,the peasant prince,201604,04\n20171018,daphne caruana galizias son accuses malta pm of complicity,201710,10\n20151012,barns risky detention policy,201510,10\n20130102,under age drinking a big problem in manning great lakes,201301,01\n20150918,the rbas advice for the us fed on hiking rates,201509,09\n20151027,adelaide bite baseballer's assault charge may be dropped,201510,10\n20070207,survey normal govt procedure says minister,200702,02\n20170324,anz joins the rush to raise home loan interest rates,201703,03\n20110214,work to start on new adelaide airport parking,201102,02\n20130309,interview johnathan thurston,201303,03\n20101206,west coast abalone season winds up,201012,12\n20110705,westhoff injury gives cornes his chance,201107,07\n20100930,pyne sent from chamber for hopeless jibe,201009,09\n20120515,rocks to tackle foreshore erosion woes,201205,05\n20101217,storm threat eases in south east queensland,201012,12\n20041017,richmond slips away from anthony,200410,10\n20070910,rare nsw plant faces extinction,200709,09\n20140602,clunies ross science award for gravity separator,201406,06\n20090713,angelita pires on trial for conspiracy,200907,07\n20070916,nt comes to grips with alcohol bans,200709,09\n20040929,tourism award nomination for pioneer settlement,200409,09\n20100223,australia v west indies innings highlights,201002,02\n20080508,people must be across risks and benefits of gm,200805,05\n20080624,goodes accepts ban,200806,06\n20030619,capriati and rubin win at eastbourne,200306,06\n20100610,youth job agency to close doors,201006,06\n20051110,call made to cut infrastructure project red tape,200511,11\n20130530,adam scott not planning to sue over anchoring,201305,05\n20041216,toxicologist calls for more drink spiking evidence,200412,12\n20110605,police find teen detention centre escapee,201106,06\n20060727,memorial to honour murdered sisters,200607,07\n20150908,jason day heads presidents cup team to take on us in october,201509,09\n20040702,icc confirms postponement of zimbabwe tests,200407,07\n20120413,philips bob brown,201204,04\n20080318,newcastle building society passes on rate rise,200803,03\n20121121,emma roberts avery wines,201211,11\n20101218,vics take innings points,201012,12\n20130514,nt cattle sold to vic,201305,05\n20101122,art world welcomes indigenous recruits,201011,11\n20130227,hough eyeing off moscow berth,201302,02\n20120718,an thai military outpost and village attacked,201207,07\n20110331,labors downfall the machine and the split,201103,03\n20150715,tonga pm casts doubt on country's ability to host pacific games,201507,07\n20141002,accc approves sale of acttab to tabcorp group,201410,10\n20050930,hope for power station to attract new industries,200509,09\n20140317,hamelin wake,201403,03\n20101013,11 jailed over van gogh theft,201010,10\n20090418,20 hostages freed from pirate mother ship,200904,04\n20131121,probe into 2011 police shooting in coffs harbour still incomple,201311,11\n20090920,torres double gets liverpool home,200909,09\n20100502,mayfair holding firm at quail hollow,201005,05\n20041106,samarra car bombs kill 8 wound 20,200411,11\n20080923,ses under pressure as storms hit riverina,200809,09\n20150528,australians unaware they have chronic kidney disease report,201505,05\n20080929,court hears torres strait seas claim,200809,09\n20141118,abortion row erupts between coalition candidates in ballarat,201411,11\n20090211,tornado kills 8 people in oklahoma,200902,02\n20170623,danny noonan ex afl player jailed for stealing from clients,201706,06\n20151104,efficient housing a focus for aboriginal land council's new w,201511,11\n20070416,missing elderly man found safe,200704,04\n20060607,council includes road repair funds in draft budget,200606,06\n20090903,cba feels wrath over storm collapse,200909,09\n20121209,marquez knocks out pacquiao,201212,12\n20090619,sharks fraud claims parents charged,200906,06\n20121219,ambulance reforms written off by paramedic's union,201212,12\n20151221,water sharing arrangement could be fast tracked due to contamin,201512,12\n20070514,viduka in no rush to decide future,200705,05\n20100212,penn universitys climategate findings,201002,02\n20051014,bikers ride honours sheene,200510,10\n20090201,hotter drier january,200902,02\n20091231,capital fireworks to bring in new year,200912,12\n20150327,joeys to be released into the wild after adelaide bushfires,201503,03\n20100223,amcor profit beats expectations,201002,02\n20040813,sex charges highlight need for workplace education,200408,08\n20030326,libs claim south coast seat,200303,03\n20060502,federal govt to fund airport security upgrade,200605,05\n20100710,yacht murder case begins,201007,07\n20070305,carpenter vows to force grill out of alp,200703,03\n20051006,us senate moves to ban prisoner torture,200510,10\n20121223,tendulkar retires from odis,201212,12\n20141003,nobel peace summit 'suspended' over dalai lama visa row,201410,10\n20050601,schumacher dismisses quit questions,200506,06\n20040921,parents shy away from meningococcal vaccinations,200409,09\n20121023,prince charles australian travel plans revealed,201210,10\n20140509,new mental health centre to help patients,201405,05\n20030929,lisbie hat trick stuns liverpool,200309,09\n20060202,awb kickbacks scandal puts govt under us pressure,200602,02\n20050909,man killed in head on crash,200509,09\n20130725,nrn ag minister shepp,201307,07\n20070807,croydon council delivers budget,200708,08\n20121102,an worldbank earmarks $245m for burma,201211,11\n20110523,doubt behind the aggression,201105,05\n20100826,interview brett kimmorley,201008,08\n20040703,new disease threatens qld citrus crops,200407,07\n20080522,man charged with assaulting girls wanted in qld,200805,05\n20140709,mining ojbection legislation changes,201407,07\n20160308,efforts to get more women to become truck drivers in tasmania,201603,03\n20040706,crackdown on overseas trained country doctors,200407,07\n20151119,national rural news,201511,11\n20140321,sydney light rail extension to open next week,201403,03\n20151211,doris fenbows killer alexis katsis jailed for 15 years,201512,12\n20111012,waca ceo wood resigns,201110,10\n20060824,program cuts childhood obesity rate researchers say,200608,08\n20140130,hospital forced to use surge capacity beds on regular basis,201401,01\n20101006,red cross opens doors in kalgoorlie boulder,201010,10\n20030716,boyle praises freeman as best of her generation,200307,07\n20131114,ract takes over federal groups' tourism ventures,201311,11\n20170529,queensland government to play ball over adani loan: treasurer,201705,05\n20151021,milky way galaxy star forming clouds,201510,10\n20120511,van egmond admits informal talks about leaving jets,201205,05\n20110718,more groundwater trials at mount zero,201107,07\n20051212,angel wins murgon by election,200512,12\n20100301,record rain fills heart of australia,201003,03\n20090727,council to sign algae biodiesel agreement,200907,07\n20121207,uninterrupted grain harvest nears end,201212,12\n20160824,wesfarmers richard goyder defends business council,201608,08\n20051017,briefings to be held for would be councillors,200510,10\n20040623,hobart prepares for jim bacons funeral,200406,06\n20070807,second suspected foot and mouth outbreak in britain,200708,08\n20101010,qr national float details unveiled,201010,10\n20060919,brock funeral begins in melbourne,200609,09\n20170620,family road trip tells burke and wills story through theatre,201706,06\n20151109,china and australia to share antarctic sea ice research,201511,11\n20141119,victoria beats south australia in shield,201411,11\n20150930,tas country hour wednesday 21 september 2015,201509,09\n20141015,consumer sentiment negative in westpac survey,201410,10\n20090719,india can make its own decisions clinton says,200907,07\n20140320,council urged to crack down on illegal holiday,201403,03\n20080925,dog attacks policewoman in boulder,200809,09\n20080123,springborg attempting to rebadge the national,200801,01\n20050120,houses crack in canadian cold spell,200501,01\n20130923,mining company discovers second cement spill in sugarloaf,201309,09\n20031108,us jobs figures fail to bolster markets,200311,11\n20110519,boat tragedy video released,201105,05\n20121102,call for review of water concessions,201211,11\n20120616,interview michael maguire,201206,06\n20030413,death toll rises on nsw roads,200304,04\n20110330,no verdict in airport caterer drug case,201103,03\n20100921,study to probe field days value,201009,09\n20100912,resilience will help say dogs,201009,09\n20110607,boaties rescue sparks emergency beacon reminder,201106,06\n20110628,robinson re signs with reds,201106,06\n20040110,fleming ton seals kiwi win,200401,01\n20111123,holden recalls diesel cars,201111,11\n20041012,china may sign fta with nz first,200410,10\n20130417,new radar,201304,04\n20140304,nsw country hour 4 march 2014,201403,03\n20060807,stanhope rejects tax discrepancy claims,200608,08\n20070308,downpour cancels bemboka show,200703,03\n20160718,toowoomba south lnp david janetzki claims victory in by election,201607,07\n20101208,flood peak fears ease in rockhampton,201012,12\n20050525,dumped car not linked to missing schoolboy police,200505,05\n20071115,second stage of vegie industry water saving,200711,11\n20080908,aust paralympic swimmers miss out on medals,200809,09\n20150622,geelong star kills another dolphin prompting fishery closure,201506,06\n20050417,ofc backs socceroos asian move,200504,04\n20150316,islamic state militants claim attack on checkpoint in libya,201503,03\n20080729,luhrmann on transformative experiences,200807,07\n20111115,man jailed over beer bottle glassing,201111,11\n20051031,windies coach denies players have attitude problem,200510,10\n20101119,court jails driver for running down man,201011,11\n20110503,pakistan embarrassed by intelligence failure,201105,05\n20071121,security camera funding pledge for mackay,200711,11\n20110104,police suspect careless campers behind bushfire,201101,01\n20150825,san francisco coach attempts to hose down hayne hype,201508,08\n20030315,hewitt still top dog,200303,03\n20131227,ukraine protesters rally after journalist bashed,201312,12\n20080423,bryce bligh address students at brisbane anzac,200804,04\n20080902,domestic markets flat despite interest rate cut,200809,09\n20080113,bligh approval soars to 68pc,200801,01\n20080303,southern road fatality,200803,03\n20160127,tunarama 2016 highlights port lincoln,201601,01\n20141223,warner will be ready for boxing day test,201412,12\n20150707,75yo fraser coast woman dies after suspected,201507,07\n20090515,rees urges players to come forward,200905,05\n20140311,smith agrees to four year extension at storm,201403,03\n20120511,black caviar prepares for australian finale,201205,05\n20160929,sa weather fuel shortages eyre peninsula residents stranded,201609,09\n20151209,north coast victims tell stolen generations inquiry more suppor,201512,12\n20141204,ebola global toll rises further as virus spreads in sierra leone,201412,12\n20071101,bryan cousins lashes out at media,200711,11\n20070211,clashes flare again over jerusalem mosque,200702,02\n20101220,blisters and pimples clog 000,201012,12\n20140731,australian medical association regional queenslanders obese,201407,07\n20080213,apology welcome reconciliation the next goal tas,200802,02\n20050916,two injured in skydiving accident,200509,09\n20151211,captain of honduras soccer team shot dead,201512,12\n20090102,israels labour rebounds in polls after gaza blitz,200901,01\n20111115,karumba barra centre could close,201111,11\n20090826,nelson proud of saving propellant factory,200908,08\n20130330,couple wanted over sydney diamond heist,201303,03\n20090501,mp demands more police to fill shortages,200905,05\n20141010,glenn hall re signs with north queensland cowboys,201410,10\n20140425,projections illuminate anzacs,201404,04\n"))
df['parse'] = df.headline_text.apply(whitespace_nlp_with_sentences)
df['publish_yearmonth'] = df['publish_yearmonth'].astype(str)
df['publish_month'] = df['publish_month'].astype(str)
cls.corpus = (CorpusFromParsedDocuments(df,
category_col='publish_yearmonth',
parsed_col='parse')
.build())
def build(self):
'''
:return: ParsedCorpus
'''
category_col = 'Category'
while category_col in self.df:
category_col = 'Category_' + ''.join(np.random.choice(string.ascii_letters) for _ in range(5))
return CorpusFromParsedDocuments(
self.df.assign(**{category_col: '_'}), category_col, self.parsed_col
).build()
def setUp(cls):
cls.categories, cls.documents = get_docs_categories()
cls.parsed_docs = []
for doc in cls.documents:
cls.parsed_docs.append(whitespace_nlp(doc))
cls.df = pd.DataFrame({
'category':
cls.categories,
'author': ['a', 'a', 'c', 'c', 'c', 'c', 'd', 'd', 'e', 'e'],
'parsed':
cls.parsed_docs,
'document_lengths': [len(doc) for doc in cls.documents]
})
cls.corpus = CorpusFromParsedDocuments(cls.df, 'category',
'parsed').build()
def main():
nlp = spacy.load('en')
#nlp = whitespace_nlp_with_sentences
convention_df = SampleCorpora.ConventionData2012.get_data()
convention_df['parsed'] = convention_df.text.apply(nlp)
corpus = (CorpusFromParsedDocuments(convention_df,
category_col='party',
parsed_col='parsed')
.build()
.get_unigram_corpus())
model = word2vec.Word2Vec(size=100,
alpha=0.025,
window=5,
min_count=5,
max_vocab_size=None,
sample=0,
seed=1,
workers=1,
min_alpha=0.0001,
sg=1,
hs=1,
negative=0,
cbow_mean=0,
iter=10,
null_word=0,
trim_rule=None,
sorted_vocab=1)
html = word_similarity_explorer_gensim(corpus,
category='democrat',
target_term='jobs',
category_name='Democratic',
not_category_name='Republican',
minimum_term_frequency=5,
width_in_pixels=1000,
metadata=convention_df['speaker'],
word2vec=Word2VecFromParsedCorpus(corpus, model).train(),
term_significance=ScaledFScoreSignificance(),
max_p_val=0.05,
save_svg_button=True,
d3_url='scattertext/data/viz/scripts/d3.min.js',
d3_scale_chromatic_url='scattertext/data/viz/scripts/d3-scale-chromatic.v1.min.js')
open('./demo_gensim_similarity.html', 'wb').write(html.encode('utf-8'))
print('Open ./demo_gensim_similarity.html in Chrome or Firefox.')
def main():
nlp = spacy.en.English()
convention_df = SampleCorpora.ConventionData2012.get_data()
convention_df['parsed'] = convention_df.text.apply(nlp)
corpus = CorpusFromParsedDocuments(convention_df,
category_col='party',
parsed_col='parsed').build()
model = word2vec.Word2Vec(size=300,
alpha=0.025,
window=5,
min_count=5,
max_vocab_size=None,
sample=0,
seed=1,
workers=1,
min_alpha=0.0001,
sg=1,
hs=1,
negative=0,
cbow_mean=0,
iter=1,
null_word=0,
trim_rule=None,
sorted_vocab=1)
html = word_similarity_explorer_gensim(corpus,
category='democrat',
category_name='Democratic',
not_category_name='Republican',
target_term='jobs',
minimum_term_frequency=5,
pmi_filter_thresold=4,
width_in_pixels=1000,
metadata=convention_df['speaker'],
word2vec=Word2VecFromParsedCorpus(
corpus, model).train(),
max_p_val=0.1,
save_svg_button=True)
open('./demo_gensim_similarity.html', 'wb').write(html.encode('utf-8'))
print('Open ./demo_gensim_similarity.html in Chrome or Firefox.')
from scattertext.CorpusFromParsedDocuments import CorpusFromParsedDocuments
from scattertext import SampleCorpora, whitespace_nlp_with_sentences, produce_scattertext_explorer
convention_df = SampleCorpora.ConventionData2012.get_data().assign(
parse=lambda df: df.text.apply(whitespace_nlp_with_sentences))
corpus = CorpusFromParsedDocuments(convention_df,
category_col='party',
parsed_col='parse').build()
html = produce_scattertext_explorer(
corpus,
category='democrat',
category_name='Democratic',
not_category_name='Republican',
minimum_term_frequency=5,
pmi_threshold_coefficient=8,
width_in_pixels=1000,
metadata=convention_df['speaker'],
d3_scale_chromatic_url=
'scattertext/data/viz/scripts/d3-scale-chromatic.v1.min.js',
d3_url='scattertext/data/viz/scripts/d3.min.js',
)
open('./demo.html', 'wb').write(html.encode('utf-8'))
print('Open ./demo.html in Chrome or Firefox.')
def cleanhtml(raw_html):
cleanr = re.compile('<.*?>')
cleantext = re.sub(cleanr, '', str(raw_html))
return cleantext
df['clean_article_text'] = df['article_text'].apply(cleanhtml)
df.drop('article_text', axis=1, inplace=True)
df_1 = df.groupby( [ "author"] ).size().reset_index(name='Counts')
df_1 = df_1.sort_values(by=['Counts'], ascending=False)
df_1 = df_1.head(2)
df_2 = df_1.merge(df, on='author', how='inner')
df_2 = df_2.sort_values(by=['Counts'], ascending=False)
df_2['parsed'] = df_2.clean_article_text.apply(nlp)
build_corpus = CorpusFromParsedDocuments(df_2, category_col='author', parsed_col='parsed').build()
build_model = word2vec.Word2Vec(size=300,alpha=0.025,window=5,min_count=5,max_vocab_size=None,
sample=0,
seed=1,
workers=1,
min_alpha=0.0001,
sg=1,
hs=1,
negative=0,
cbow_mean=0,
iter=1,
null_word=0,
trim_rule=None,
sorted_vocab=1)