def getComplaints(self, year, productid, product):
content = download.getFile("http://old.asa.co.nz/srch_product.php", download.getFilename(year, "product", product), self.refreshYear(year), {'product_id': productid, 'year': year})
if content:
complaints = re.finditer(ur"<tr align='left' valign='top'><td><b><p><a href='display.php\?ascb_number=(?P<id>[0-9]{5})'>[0-1][0-9]/[0-9]{3}(?:.*?)</b> - (?P<advert>.*?)</a></p></td><td><p>(?P<decision>.*?)</p></td><td><a href='(?P<docurl>.*?)'><p>Full Decision</p></a></td></tr>", content, flags=re.DOTALL)
for complaintmatch in complaints:
self.complaint.complaint = complaintmatch.groupdict()
self.complaint.complaint["product"] = product
self.getComplaint(self.refreshYear(year))
def get(self):
if "id" in self.complaint and self.complaint["id"]:
self.complaint["year"] = int("20" + self.complaint["id"][0:2])
docType = self.docType(self.complaint["year"])
docfilename = download.getFilename(self.complaint["year"], "docs", self.complaint["id"], "." + docType)
textfilename = download.getFilename(self.complaint["year"], "text", self.complaint["id"], ".txt")
htmlfilename = download.getFilename(self.complaint["year"], "html", self.complaint["id"], ".html")
#download.getFile("/".join(("http://203.152.114.11/decisions", self.complaint["id"][0:2], self.complaint["id"] + "." + docType)), docfilename, self.refresh, returnfile = False)
download.getFile("http://old.asa.co.nz/decision_file.php?ascbnumber=" + self.complaint["id"], docfilename, self.refresh, returnfile = False)
if docType == "doc":
self.__getPage(download.getFile("http://old.asa.co.nz/display.php?ascb_number=" + self.complaint["id"], download.getFilename(self.complaint["year"], 'pages', self.complaint["id"]), self.refresh))
for field in ["docdate", "docsize", "docwords", "complainants", "companies", "meetingdate"]:
self.complaint[field] = None
self.complaint["docdate"], self.complaint["docsize"] = download.getFileDetails(docfilename)
if os.path.exists(docfilename):
if os.path.exists(textfilename) and (self.complaint["docdate"] == os.path.getmtime(textfilename) or self.quick):
self.complaint["doc"] = download.loadResource(textfilename)
else:
if docType == "doc":
command = [antiword, "-w", "0", "-m", "8859-1.txt"]
else:
command = [unrtf, "--text"]
command.append(os.path.join(os.getcwd(), docfilename))
try:
self.complaint["doc"] = subprocess.check_output(command).decode('unicode_escape')
except Exception, e:
print "Failed to convert doc " + os.path.join(os.getcwd(), docfilename) + ": " + str(e)
else:
if docType == "rtf":
self.complaint["doc"] = self.complaint["doc"].split("-----------------", 1)[1]
download.saveResourcePost(textfilename, self.complaint["doc"], (time.time(), self.complaint["docdate"]))
if os.path.exists(htmlfilename) and (self.complaint["docdate"] == os.path.getmtime(htmlfilename) or self.quick):
self.complaint["html"] = download.loadResource(htmlfilename)
else:
command2 = [unoconv, "-f", "html", "-o", os.path.join(os.getcwd(), htmlfilename), os.path.join(os.getcwd(), docfilename)]
print "Saving: " + htmlfilename
subprocess.check_output(command2)
self.complaint["html"] = re.search('<body(?:.*?)>(.*?)</body>', download.loadResource(htmlfilename), flags=re.DOTALL).group(1)
def __fill(self):
decisions = list()
decisions.append(("Chairmans Ruling: Complaint Resolved", "Chairman", "Resolved", 1))
decisions.append(("Chairmans Ruling: Complaint Settled", "Chairman", "Settled", 1))
decisions.append(("Chairmans Ruling: Complaint Withdrawn", "Chairman", "Withdrawn", None))
decisions.append(("Chairmans Ruling: No Grounds To Proceed", "Chairman", "No Grounds To Proceed", 0))
decisions.append(("Chairmans Ruling: No Jurisdiction", "Chairman", "No Jurisdiction", 0))
decisions.append(("Chairmans Ruling: Settled - Advertiser Error", "Chairman", "Settled - Advertiser Error", 1))
decisions.append(("Chairmans Ruling: Settled - Media Error", "Chairman", "Settled - Media Error", 1))
decisions.append(("Chairmans Ruling: Settled - Retailer Error", "Chairman", "Settled - Retailer Error", 1))
decisions.append(("Decision: Complaint No Determination", "Board", "No Determination", None))
decisions.append(("Decision: Complaint Not Upheld", "Board", "Not Upheld", 0))
decisions.append(("Decision: Complaint Resolved", "Board", "Resolved", 1))
decisions.append(("Decision: Complaint Settled", "Board", "Settled", 1))
decisions.append(("Decision: Complaint Upheld", "Board", "Upheld", 1))
decisions.append(("Decision: No Grounds To Proceed", "Board", "No Grounds To Proceed", 0))
decisions.append(("Decision: No Jurisdiction", "Board", "No Jurisdiction", 0))
decisions.append(("Decision: Complaint Adjourned (Sine Die)", "Board", "Adjourned (Sine Die)", None))
decisions.append(("See notes", None, "See Notes", None))
decisions.append(("Pre-ASCB", None, "Pending", None))
decisions.append(("Pre-Accepted", None, "Pending", None))
decisions.append(("Chairmans Ruling: Complaint Not Accepted", "Chairman", "Not Accepted", 0))
decisions.append(("Chairmans Ruling: Complaints Not Accepted", "Chairman", "Not Accepted", 0))
decisions.append(("Chairman's Ruling: Complaint Settled", "Chairman", "Settled", 1))
decisions.append(("Ruling: Complaint Settled", "Chairman", "Settled", 1))
decisions.append(("Decision: Upheld (in part)", "Board", "Upheld (in part)", 1))
decisions.append(("Decision: Complaint Upheld (in part)", "Board", "Upheld (in part)", 1))
decisions.append(("Decision: Complaint Upheld In Part", "Board", "Upheld (in part)", 1))
decisions.append(("Decision: Complaint Upheld in part and Settled in part", "Board", "Upheld (in part) and Settled (in part)", 1))
decisions.append(("Decision: Complaint Upheld (in part) Settled (in part)", "Board", "Upheld (in part) and Settled (in part)", 1))
decisions.append(("Decision: Complaint Settled (in part) Not Upheld (in part) Upheld (in part)", "Board", "Upheld (in part), Settled (in part) and Not Upheld (in part)", 1))
decisions.append(("Decision: Complaint Settled (in part)", "Board", "Settled (in part)", 1))
decisions.append(("Decision: Complaint Not Accepted", "Board", "Not Accepted", 0))
decisions.append(("Decision: Complaint Settled (in part) / Not Upheld (in part)", "Board", "Settled (in part) and Not Upheld (in part)", 1))
decisions.append(("Panel's Ruling: Complaint Settled", "Board", "Settled", 1))
#Pre-Accepted
#Pre-ASCB
#Withdrawn
#Upheld
#Settled
#Resolved
#Not Upheld
#Adjourned
#No Determination
#No Grounds To Proceed
#No Jurisdiction
#
#
#Decision: Complaint: 01/29 & 01/31 - Settled
#Decision: Complaint 01/81 Upheld
#Decision: Complaint: Not Upheld
self.cursor.executemany("INSERT INTO decisions (name, arbiter, ruling, success) VALUES (%s, %s, %s, %s);", decisions)
self.cursor.executemany("INSERT INTO products (id, name) VALUES (%s, %s);", re.findall(ur'<OPTION value="([0-9]{1,2})">(.*?)</OPTION>', download.getFile("http://old.asa.co.nz/search_product.php", download.getFilename(self.year, 'categories', "search_product")), flags=re.DOTALL))
#Upheld
#Settled
#Resolved
#Not Upheld
#Adjourned
#No Determination
#No Grounds To Proceed
#No Jurisdiction
#
#
#Decision: Complaint: 01/29 & 01/31 - Settled
#Decision: Complaint 01/81 Upheld
#Decision: Complaint: Not Upheld
self.cursor.executemany("INSERT INTO decisions (name, arbiter, ruling, success) VALUES (%s, %s, %s, %s);", decisions)
self.cursor.executemany("INSERT INTO products (id, name) VALUES (%s, %s);", re.findall(ur'<OPTION value="([0-9]{1,2})">(.*?)</OPTION>', download.getFile("http://old.asa.co.nz/search_product.php", download.getFilename(self.year, 'categories', "search_product")), flags=re.DOTALL))
self.cursor.executemany("INSERT INTO media (id, name) VALUES (%s, %s);", re.findall(ur'<OPTION value="([0-9]{1,2})">(.*?)</OPTION>', download.getFile("http://old.asa.co.nz/search_media.php", download.getFilename(self.year, 'categories', "search_media")), flags=re.DOTALL))
codes = re.findall(ur'<OPTION value="([0-9]{1,2})">(.*?)</OPTION>', download.getFile("http://old.asa.co.nz/search_code.php", download.getFilename(self.year, 'categories', "search_code")), flags=re.DOTALL)
self.cursor.executemany("INSERT INTO codes (id, name) VALUES (%s, %s);", codes)
self.cursor.executemany("UPDATE codes SET url = %s WHERE name = %s", [("http://old.asa.co.nz/code_ethics.php", "Code of Ethics"), ("http://old.asa.co.nz/code_financial.php", "Code for Financial Advertising"), ("http://old.asa.co.nz/code_people.php", "Code for People in Advertising"), ("http://old.asa.co.nz/code_weight.php", "Code for Advertising of Weight Management"), ("http://old.asa.co.nz/code_gaming.php", "Code for Gaming and Gambling"), ("http://old.asa.co.nz/code_vehicles.php", "Code for Advertising Vehicles"), ("http://old.asa.co.nz/code_therapeutic_services.php", "Therapeutic Services Advertising Code"), ("http://old.asa.co.nz/code_therapeutic_products.php", "Therapeutic Products Advertising Code"), ("http://old.asa.co.nz/code_children.php", "Code for Advertising to Children"), ("http://old.asa.co.nz/code_children_food.php", "Children's Code for Advertising Food"), ("http://old.asa.co.nz/code_food.php", "Code for Advertising Food"), ("http://old.asa.co.nz/code_promo_advert_liquor.php", "Code for Advertising and Promotion of Alcohol"), ("http://old.asa.co.nz/code_comparative.php", "Code for Comparative Advertising"), ("http://old.asa.co.nz/code_environmental.php", "Code for Environmental Claims")])
for code in codes:
for clause in re.findall(ur'<OPTION value="([0-9]{1,3})">(.*?)</OPTION>', download.getFile("http://old.asa.co.nz/srch_code.php", download.getFilename(self.year, 'categories', code[1]), post = {"code_id": code[0], "year": self.year}), flags=re.DOTALL):
self.cursor.execute("INSERT INTO clauses (id, name, codes_id) VALUES (%s, %s, %s);", clause + (code[0],))
self.loadCompanies()
self.loadComplainants()
def __empty(self):
self.cursor.execute("SET foreign_key_checks = 0")
for table in self.tables:
self.cursor.execute("EMPTY TABLE " + table)
self.cursor.execute("SET foreign_key_checks = 1")
