def try_to_link(self, vendor_name): """TODO.""" search_results = self.search_sos(vendor_name) total_hits = self.get_total_hits(search_results) if total_hits == 1: log.info("Perfect match for %s", vendor_name) self.process_direct_hit(search_results, vendor_name)
def _match_contract(self, document): ''' Match a particular contract. TODO: Better description :params document: A Python-DocumentCloud object representing a contract :type document: Python-DocumentCloud object. ''' log.info('Syncing document %s', document.id) fields = {} fields['purchaseno'] = self._get_metadata(document, "purchase order") fields['contractno'] = self._get_metadata(document, "contract number") fields['vendor'] = self._get_metadata(document, "vendor").replace(".", "") fields['department'] = self._get_metadata(document, "vendor").replace(".", "") fields['dateadded'] = document.created_at fields['title'] = document.title fields['description'] = document.description LensDatabase().add_department(fields['department']) LensDatabase().add_vendor(fields['vendor']) fields['department'] = LensDatabase().get_department_id( fields['department']) fields['vendor'] = LensDatabase().get_lens_vendor_id(fields['vendor']) LensDatabase().update_contract_from_document_cloud(document.id, fields)
def _get_attachment_display_name(self, city_attachment_id): '''docstring''' response = urllib2.urlopen( 'http://www.purchasing.cityofno.com/bso/external/document/' + 'attachments/attachmentFileDetail.sdo?' + 'fileNbr=%s' % city_attachment_id + '&docId=%s' % self.purchaseorder + '&docType=P&releaseNbr=0&parentUrl=/external/purchaseorder/' + 'poSummary.sdo&external=true' ) html = response.read() file_location = '%s/%s.html' % (ATTACHMENTS_DIR, city_attachment_id) if not os.path.exists(os.path.dirname(file_location)): os.makedirs(os.path.dirname(file_location)) with open(file_location, 'w') as filename: log.info('Saving HTML for attachment %s', city_attachment_id) filename.write(html) soup = BeautifulSoup(html) header = soup.select(".sectionheader-01")[0].contents.pop() header = ' '.join(header.split()) attachment_file_name = str(header).replace( "Attachment File Detail:", "").strip() return attachment_file_name
def _match_contract(self, document): ''' Match a particular contract. TODO: Better description :params document: A Python-DocumentCloud object representing a contract :type document: Python-DocumentCloud object. ''' log.info('Syncing document %s', document.id) fields = {} fields['purchaseno'] = self._get_metadata(document, "purchase order") fields['contractno'] = self._get_metadata(document, "contract number") fields['vendor'] = self._get_metadata(document, "vendor").replace(".", "") fields['department'] = self._get_metadata(document, "vendor").replace(".", "") fields['dateadded'] = document.created_at fields['title'] = document.title fields['description'] = document.description LensDatabase().add_department(fields['department']) LensDatabase().add_vendor(fields['vendor']) fields['department'] = LensDatabase().get_department_id(fields['department']) fields['vendor'] = LensDatabase().get_lens_vendor_id(fields['vendor']) LensDatabase().update_contract_from_document_cloud(document.id, fields)
def _get_vendor_name(self): # , soup): ''' Find the vendor name in the contract HTML. If that fails, then ___ :param soup: A BeautifulSoup object for the contract page HTML. :type soup: BeautifulSoup object. :returns: string. The contract vendor's name. ''' vendor_file_location = '%s/%s.html' % (VENDORS_DIR, self.vendor_id_city) # Downloaded this file in _download_vendor_profile() with open(vendor_file_location, 'r') as myfile: log.info('Reading HTML for vendor %s', self.vendor_id_city) html = myfile.read() soup = BeautifulSoup(html) vendor_row = soup(text='Company Name:')[0].parent.parent vendor_name = (vendor_row.findChildren(['td' ])[5].contents.pop().strip()) # Convert to uppercase for DocumentCloud project metadata. # Search queries are also converted to uppercase. vendor_name = vendor_name.upper() return vendor_name
def _backup(self, document_cloud_id): '''Backup a contract.''' needs_backup = self._needs_to_be_backed_up(document_cloud_id) if needs_backup or self.force: log.info("Creating backup for %s", document_cloud_id) document = self.client.documents.get(document_cloud_id) metadata = self._get_meta_data(document) pdf_path = self._get_path(document_cloud_id, ".pdf") pdf_exists = os.path.exists(pdf_path) if not pdf_exists or self.force: pdf = document.pdf with open(pdf_path, "wb") as outfile: outfile.write(pdf) txt_path = self._get_path(document_cloud_id, ".txt") txt_exists = os.path.exists(txt_path) if not txt_exists or self.force: with open(txt_path, "wb") as outfile: outfile.write(json.dumps(metadata)) text_txt_path = self._get_path(document_cloud_id, "_text.txt") text_txt_exists = os.path.exists(text_txt_path) if not text_txt_exists or self.force: with open(text_txt_path, "wb") as outfile: outfile.write(json.dumps(document.full_text)) else: log.info("%s is already is backed up", document_cloud_id)
def add_name(self, name): """TODO.""" name = name.replace(".", "").strip() if self.is_this_a_person(name): # people with Jr ect at the end of the name are people indb = (SESSION.query(Person).filter(Person.name == name).count()) if indb == 0: SESSION.add(Person(name)) SESSION.commit() return if indb == 1: SESSION.close() return if self._is_this_a_company(name): indb = (SESSION.query(Company).filter( Company.name == name).count()) if indb == 0: SESSION.add(Company(name)) SESSION.commit() return if indb == 1: SESSION.close() return log.info("Could not link %s", name) SESSION.close()
def __init__(self, purchase_order_number): self.purchaseorder = purchase_order_number validity = Utilities().check_if_valid_purchase_order_format( self.purchaseorder) if validity is False: log.debug('Purchase order %s is invalid', self.purchaseorder) return html = self._get_html() self.vendor_id_city = self._get_city_vendor_id(html) self._download_vendor_profile(self.vendor_id_city) soup = BeautifulSoup(html) self.description = self._get_description(soup) try: self.vendor_name = self._get_vendor_name() except IOError as e: log.error(e, exc_info=True) self.vendor_name = "unknown" log.info('No vendor info for purchase order %s', self.purchaseorder) self.department = self._get_department(soup) self.k_number = self._get_knumber(soup) self.attachments = self._get_attachments(soup) self.data = self._get_data() self.title = "%s : %s" % (self.vendor_name, self.description)
def check_if_need_to_scrape(self, page): """ If page is <= 10 and last scrape was before today, scrape it. If page is > 10 and last scrape was more than seven days ago, scrape. :params page: The purchasing site page number to check. :type page: int. :returns: boolean. True if need to scrape, False if not. """ today_date = date.today() week_ago_date = date.today() - timedelta(days=7) date_last_scraped = self._check_when_last_scraped(page) if date_last_scraped is None: return True # Scrape this page elif page <= 10: if date_last_scraped < today_date: return True # Scrape this page else: log.info('Skipping page %d. It was scraped recently', page) return False elif page > 10: if date_last_scraped < week_ago_date: return True # Scrape this page else: log.info('Skipping page %d. It was scraped recently', page) return False
def try_to_link(self, vendor_name): """TODO.""" search_results = self.search_sos(vendor_name) total_hits = self.get_total_hits(search_results) if total_hits == 1: log.info("Perfect match for %s", vendor_name) self.process_direct_hit(search_results, vendor_name)
def _backup(self, document_cloud_id): '''Backup a contract.''' needs_backup = self._needs_to_be_backed_up(document_cloud_id) if needs_backup or self.force: log.info("Creating backup for %s", document_cloud_id) document = self.client.documents.get(document_cloud_id) metadata = self._get_meta_data(document) pdf_path = self._get_path(document_cloud_id, ".pdf") pdf_exists = os.path.exists(pdf_path) if not pdf_exists or self.force: pdf = document.pdf with open(pdf_path, "wb") as outfile: outfile.write(pdf) txt_path = self._get_path(document_cloud_id, ".txt") txt_exists = os.path.exists(txt_path) if not txt_exists or self.force: with open(txt_path, "wb") as outfile: outfile.write(json.dumps(metadata)) text_txt_path = self._get_path(document_cloud_id, "_text.txt") text_txt_exists = os.path.exists(text_txt_path) if not text_txt_exists or self.force: with open(text_txt_path, "wb") as outfile: outfile.write(json.dumps(document.full_text)) else: log.info("%s is already is backed up", document_cloud_id)
def __init__(self, purchase_order_number): self.purchaseorder = purchase_order_number validity = Utilities().check_if_valid_purchase_order_format( self.purchaseorder) if validity is False: log.debug('Purchase order %s is invalid', self.purchaseorder) return html = self._get_html() self.vendor_id_city = self._get_city_vendor_id(html) self._download_vendor_profile(self.vendor_id_city) soup = BeautifulSoup(html) self.description = self._get_description(soup) try: self.vendor_name = self._get_vendor_name() except IOError as e: log.error(e, exc_info=True) self.vendor_name = "unknown" log.info('No vendor info for purchase order %s', self.purchaseorder) self.department = self._get_department(soup) self.k_number = self._get_knumber(soup) self.attachments = self._get_attachments(soup) self.data = self._get_data() self.title = "%s : %s" % (self.vendor_name, self.description)
def _download_attachment(self, attachment): ''' Download an attachment associated with a purchase order. :param attachment: The name of the attachment file to download. :type attachment: string ''' # The city's purchasing site has an internal ID for each attachment. # Here we use it to download the attachment files, and also to store # locally so we can have a list of the attachments we have on hand. city_attachment_id = re.search( '[0-9]+', attachment.get('href')).group() log.debug('Gathering data for attachment %s', city_attachment_id) document_path = '%s/%s.pdf' % (DOCUMENTS_DIR, city_attachment_id) display_name = self._get_attachment_display_name(city_attachment_id) if os.path.isfile(document_path): # Have already downloaded log.info('Already have PDF for attachment %s', city_attachment_id) else: self._download_attachment_file(city_attachment_id, display_name, document_path)
def _get_attachments(soup): ''' Find the attachments to download from the contract page HTML. :param soup: A BeautifulSoup object for the contract page's HTML. :type soup: BeautifulSoup object :returns: ??? ''' try: main_table = soup.select('.table-01').pop() metadatarow = (main_table .findChildren(['tr'])[2] .findChildren(['td'])[0] .findChildren(['table'])[0] .findChildren(['tr'])) attachment_filenames = (metadatarow[16] .findChildren(['td'])[1] .findChildren(['a'])) return attachment_filenames except IndexError: log.info('No attachments found') return [] # The city does not always include attachment files.
def _get_vendor_name(self): # , soup): ''' Find the vendor name in the contract HTML. If that fails, then ___ :param soup: A BeautifulSoup object for the contract page HTML. :type soup: BeautifulSoup object. :returns: string. The contract vendor's name. ''' vendor_file_location = '%s/%s.html' % (VENDORS_DIR, self.vendor_id_city) # Downloaded this file in _download_vendor_profile() with open(vendor_file_location, 'r') as myfile: log.info('Reading HTML for vendor %s', self.vendor_id_city) html = myfile.read() soup = BeautifulSoup(html) vendor_row = soup(text='Company Name:')[0].parent.parent vendor_name = (vendor_row .findChildren(['td'])[5] .contents.pop().strip()) # Convert to uppercase for DocumentCloud project metadata. # Search queries are also converted to uppercase. vendor_name = vendor_name.upper() return vendor_name
def _get_attachment_display_name(self, city_attachment_id): '''docstring''' response = urllib2.urlopen( 'http://www.purchasing.cityofno.com/bso/external/document/' + 'attachments/attachmentFileDetail.sdo?' + 'fileNbr=%s' % city_attachment_id + '&docId=%s' % self.purchaseorder + '&docType=P&releaseNbr=0&parentUrl=/external/purchaseorder/' + 'poSummary.sdo&external=true') html = response.read() file_location = '%s/%s.html' % (ATTACHMENTS_DIR, city_attachment_id) if not os.path.exists(os.path.dirname(file_location)): os.makedirs(os.path.dirname(file_location)) with open(file_location, 'w') as filename: log.info('Saving HTML for attachment %s', city_attachment_id) filename.write(html) soup = BeautifulSoup(html) header = soup.select(".sectionheader-01")[0].contents.pop() header = ' '.join(header.split()) attachment_file_name = str(header).replace("Attachment File Detail:", "").strip() return attachment_file_name
def _check_if_need_to_download_contract(purchase_order_number): ''' Determines whether this contract should be downloaded, and also whether it needs to be added to our DocumentCloud and local database. :param purchase_order_number: The contract's purchase order number. :type purchase_order_number: string ''' log.info('Checking purchase order %s', purchase_order_number) # Check local file repository try: log.debug('LensRepository') need_to_download = LensRepository( purchase_order_number).check_if_need_to_download() if need_to_download: LensRepository(purchase_order_number).download_purchase_order() except urllib2.HTTPError: log.exception('Purchase order %s not posted publically', purchase_order_number) try: log.debug('PurchaseOrder') purchase_order_object = PurchaseOrder(purchase_order_number) purchase_order_object.download_attachments() except IndexError: log.exception(purchase_order_number) return # Check DocumentCloud project try: log.debug('DocumentCloudProject') need_to_upload = DocumentCloudProject().check_if_need_to_upload( purchase_order_number) if need_to_upload: DocumentCloudProject().prepare_then_add_contract( purchase_order_object) except urllib2.HTTPError: log.exception('Purchase order %s not posted publically', purchase_order_number) # Check local database try: log.debug('LensDatabase') contract_exist = LensDatabase().check_if_database_has_contract( purchase_order_number) if contract_exist is False: LensDatabase().add_to_database(purchase_order_object) except urllib2.HTTPError: log.exception('Purchase order %s is not posted publically.', purchase_order_number)
def _check_if_need_to_download_contract(purchase_order_number): ''' Determines whether this contract should be downloaded, and also whether it needs to be added to our DocumentCloud and local database. :param purchase_order_number: The contract's purchase order number. :type purchase_order_number: string ''' log.info('Checking purchase order %s', purchase_order_number) # Check local file repository try: log.debug('LensRepository') need_to_download = LensRepository( purchase_order_number).check_if_need_to_download() if need_to_download: LensRepository(purchase_order_number).download_purchase_order() except urllib2.HTTPError: log.exception('Purchase order %s not posted publically', purchase_order_number) try: log.debug('PurchaseOrder') purchase_order_object = PurchaseOrder(purchase_order_number) purchase_order_object.download_attachments() except IndexError: log.exception(purchase_order_number) return # Check DocumentCloud project try: log.debug('DocumentCloudProject') need_to_upload = DocumentCloudProject().check_if_need_to_upload( purchase_order_number) if need_to_upload: DocumentCloudProject().prepare_then_add_contract( purchase_order_object) except urllib2.HTTPError: log.exception('Purchase order %s not posted publically', purchase_order_number) # Check local database try: log.debug('LensDatabase') contract_exist = LensDatabase().check_if_database_has_contract( purchase_order_number) if contract_exist is False: LensDatabase().add_to_database(purchase_order_object) except urllib2.HTTPError: log.exception('Purchase order %s is not posted publically.', purchase_order_number)
def _get_html(self): '''Read the HTML contents of this purchase order file. :returns: string. The HTML contains for this purchase order file. ''' file_location = '%s/%s.html' % (PURCHASE_ORDER_DIR, self.purchaseorder) # Purchase order HTML saved in PurchaseOrder class with open(file_location, 'r') as html_file: log.info('Reading HTML for purchase order %s', self.purchaseorder) return html_file.read()
def _get_html(self): '''Read the HTML contents of this purchase order file. :returns: string. The HTML contains for this purchase order file. ''' file_location = '%s/%s.html' % (PURCHASE_ORDER_DIR, self.purchaseorder) # Purchase order HTML saved in PurchaseOrder class with open(file_location, 'r') as html_file: log.info('Reading HTML for purchase order %s', self.purchaseorder) return html_file.read()
def link(self, name, vendor): """Link the vendor to the company.""" name = name.strip("\n").replace(".", "").strip() # get the vendor: vendorindb = (SESSION.query(Vendor) .filter(Vendor.name == vendor) .first()) # get the person: personindb = (SESSION.query(Person) .filter(Person.name == name) .first()) co = (SESSION.query(Company) .filter(Company.name == name)) companyindb = co.first() # get the company if personindb is not None and companyindb is None: link = (SESSION.query(VendorOfficer) .filter(VendorOfficer.vendorid == vendorindb.id) .filter(VendorOfficer.personid == personindb.id) .count()) if vendorindb is not None and personindb is not None and link < 1: log.info("Linking {0} to {1}", str(vendorindb.id), str(personindb.id)) link = VendorOfficer(vendorindb.id, personindb.id) SESSION.add(link) SESSION.commit() return if companyindb is not None and personindb is None: link = (SESSION.query(VendorOfficerCompany) .filter(VendorOfficerCompany.vendorid == vendorindb.id) .filter(VendorOfficerCompany.companiesid == companyindb.id) .count()) if vendorindb is not None and companyindb is not None and link < 1: print("Linking {0} to {1}".format( str(vendorindb.id), str(companyindb.id) )) link = VendorOfficerCompany(vendorindb.id, companyindb.id) SESSION.add(link) SESSION.commit() return SESSION.close()
def _check_if_contract_number_is_null(purchase_order_object): ''' Checks if this contract number is null. :params purchase_order_object: A PurchaseOrder object. :type purchase_order_object: A PurchaseOrder object. :returns: boolean. True if the contract number is null, False if not. ''' if len(purchase_order_object.data['contract number']) < 1: log.info('Not uploading purchase order %s to DocumentCloud', purchase_order_object.data['purchase order']) log.info('Contract number %s is null', purchase_order_object.data['contract number']) return True else: return False
def match_local_database_to_document_cloud(self): ''' Match our local database to our DocumentCloud project. TODO: Why fetching half-filled contracts? ''' half_filled_contracts = LensDatabase().get_half_filled_contracts() log.info('%d half-filled contracts need to be synced', len(half_filled_contracts)) for half_filled_contract in half_filled_contracts: try: contract = self.client.documents.get( half_filled_contract.doc_cloud_id) self._match_contract(contract) except Exception as e: log.error(e, exc_info=True)
def link(self, name, vendor): """Link the vendor to the company.""" name = name.strip("\n").replace(".", "").strip() # get the vendor: vendorindb = (SESSION.query(Vendor).filter( Vendor.name == vendor).first()) # get the person: personindb = (SESSION.query(Person).filter( Person.name == name).first()) co = (SESSION.query(Company).filter(Company.name == name)) companyindb = co.first() # get the company if personindb is not None and companyindb is None: link = (SESSION.query(VendorOfficer).filter( VendorOfficer.vendorid == vendorindb.id).filter( VendorOfficer.personid == personindb.id).count()) if vendorindb is not None and personindb is not None and link < 1: log.info("Linking {0} to {1}", str(vendorindb.id), str(personindb.id)) link = VendorOfficer(vendorindb.id, personindb.id) SESSION.add(link) SESSION.commit() return if companyindb is not None and personindb is None: link = (SESSION.query(VendorOfficerCompany).filter( VendorOfficerCompany.vendorid == vendorindb.id).filter( VendorOfficerCompany.companiesid == companyindb.id).count()) if vendorindb is not None and companyindb is not None and link < 1: print("Linking {0} to {1}".format(str(vendorindb.id), str(companyindb.id))) link = VendorOfficerCompany(vendorindb.id, companyindb.id) SESSION.add(link) SESSION.commit() return SESSION.close()
def _write_purchase_order(self, html, file_location): ''' This takes an individual contract page's HTML and writes it out to an HTML file in the proper location. :param html: The individual contract page's HTML. :type html: string. :param file_location: The path to where the file should be created. :type file_location: string. ''' if not os.path.exists(os.path.dirname(file_location)): os.makedirs(os.path.dirname(file_location)) with open(file_location, 'w') as filename: log.info('Saving HTML for purchase order %s', self.purchase_order_number) filename.write(html)
def _write_purchase_order(self, html, file_location): ''' This takes an individual contract page's HTML and writes it out to an HTML file in the proper location. :param html: The individual contract page's HTML. :type html: string. :param file_location: The path to where the file should be created. :type file_location: string. ''' if not os.path.exists(os.path.dirname(file_location)): os.makedirs(os.path.dirname(file_location)) with open(file_location, 'w') as filename: log.info('Saving HTML for purchase order %s', self.purchase_order_number) filename.write(html)
def match_local_database_to_document_cloud(self): ''' Match our local database to our DocumentCloud project. TODO: Why fetching half-filled contracts? ''' half_filled_contracts = LensDatabase().get_half_filled_contracts() log.info('%d half-filled contracts need to be synced', len(half_filled_contracts)) for half_filled_contract in half_filled_contracts: try: contract = self.client.documents.get( half_filled_contract.doc_cloud_id ) self._match_contract(contract) except Exception as e: log.error(e, exc_info=True)
def get_people_associated_with_vendor(self, name): """ Get a list of people associated with the vendor. Not called on by this class, but is called on by emailer.py. :param name: The vendor name. :type name: string :returns: list. The people who are associated with this vendor (how?). """ query = (SESSION.query( Person.name).filter(Vendor.id == VendorOfficer.vendorid).filter( Person.id == VendorOfficer.personid).filter( Vendor.name == name).all()) SESSION.close() log.info('%d people associated with %s', len(query), name) return [str(row[0]) for row in query]
def check_if_database_has_contract(self, purchase_order_number): """ Check if local database already has this contract. :param purchase_order_number: The unique ID in the city's website. :type purchase_order_number: string :returns: boolean. True if the contract is present, False if not. """ count = (SESSION.query(Contract).filter( Contract.purchaseordernumber == purchase_order_number).count()) SESSION.close() if count == 1: # Database has the contract log.info('DB contracts table already has purchase order %s', purchase_order_number) return True else: log.info('DB contracts table does not have purchase order %s', purchase_order_number) return False
def _download_vendor_profile(city_vendor_id): ''' Download the vendor page associated with a purchase order, if we don't have the vendor page already. :param city_vendor_id: The vendor ID on the city's purchasing site. :type city_vendor_id: string. ''' vendor_file_location = '%s/%s.html' % (VENDORS_DIR, city_vendor_id) if os.path.isfile(vendor_file_location): log.info('Already have HTML for vendor %s', city_vendor_id) else: try: response = urllib2.urlopen( 'http://www.purchasing.cityofno.com/' + 'bso/external/vendor/vendorProfileOrgInfo.sdo?' + 'external=true&vendorId={}'.format(city_vendor_id)) html = response.read() if not os.path.exists(os.path.dirname(vendor_file_location)): os.makedirs(os.path.dirname(vendor_file_location)) with open(vendor_file_location, 'w') as filename: log.info('Saving HTML for vendor %s', city_vendor_id) filename.write(html) except urllib2.HTTPError: log.info('Could not save HTML for vendor %s', city_vendor_id)
def _download_vendor_profile(city_vendor_id): ''' Download the vendor page associated with a purchase order, if we don't have the vendor page already. :param city_vendor_id: The vendor ID on the city's purchasing site. :type city_vendor_id: string. ''' vendor_file_location = '%s/%s.html' % (VENDORS_DIR, city_vendor_id) if os.path.isfile(vendor_file_location): log.info('Already have HTML for vendor %s', city_vendor_id) else: try: response = urllib2.urlopen( 'http://www.purchasing.cityofno.com/' + 'bso/external/vendor/vendorProfileOrgInfo.sdo?' + 'external=true&vendorId={}'.format(city_vendor_id)) html = response.read() if not os.path.exists(os.path.dirname(vendor_file_location)): os.makedirs(os.path.dirname(vendor_file_location)) with open(vendor_file_location, 'w') as filename: log.info('Saving HTML for vendor %s', city_vendor_id) filename.write(html) except urllib2.HTTPError: log.info('Could not save HTML for vendor %s', city_vendor_id)
def _get_attachments(soup): ''' Find the attachments to download from the contract page HTML. :param soup: A BeautifulSoup object for the contract page's HTML. :type soup: BeautifulSoup object :returns: ??? ''' try: main_table = soup.select('.table-01').pop() metadatarow = (main_table.findChildren(['tr'])[2].findChildren( ['td'])[0].findChildren(['table'])[0].findChildren(['tr'])) attachment_filenames = (metadatarow[16].findChildren( ['td'])[1].findChildren(['a'])) return attachment_filenames except IndexError: log.info('No attachments found') return [] # The city does not always include attachment files.
def _download_attachment(self, attachment): ''' Download an attachment associated with a purchase order. :param attachment: The name of the attachment file to download. :type attachment: string ''' # The city's purchasing site has an internal ID for each attachment. # Here we use it to download the attachment files, and also to store # locally so we can have a list of the attachments we have on hand. city_attachment_id = re.search('[0-9]+', attachment.get('href')).group() log.debug('Gathering data for attachment %s', city_attachment_id) document_path = '%s/%s.pdf' % (DOCUMENTS_DIR, city_attachment_id) display_name = self._get_attachment_display_name(city_attachment_id) if os.path.isfile(document_path): # Have already downloaded log.info('Already have PDF for attachment %s', city_attachment_id) else: self._download_attachment_file(city_attachment_id, display_name, document_path)
def add_name(self, name): """TODO.""" name = name.replace(".", "").strip() if self.is_this_a_person(name): # people with Jr ect at the end of the name are people indb = (SESSION.query(Person) .filter(Person.name == name) .count()) if indb == 0: SESSION.add(Person(name)) SESSION.commit() return if indb == 1: SESSION.close() return if self._is_this_a_company(name): indb = (SESSION.query(Company) .filter(Company.name == name) .count()) if indb == 0: SESSION.add(Company(name)) SESSION.commit() return if indb == 1: SESSION.close() return log.info("Could not link %s", name) SESSION.close()
url = 'http://www.purchasing.cityofno.com/bso/' + \ 'external/advsearch/searchContract.sdo' req = urllib2.Request(url=url, data=data) req.add_header('Pragma', ' no-cache') req.add_header('Origin', 'http://www.purchasing.cityofno.com') req.add_header('Accept-Encoding', 'gzip, deflate') req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.add_header( 'Accept', 'text/add_contracthtml,application/xhtml+xml,application/xml;' + 'q=0.9,image/webp,*/*;q=0.8') req.add_header('Cache-Control', 'no-cache') req.add_header( 'Referer', 'http://www.purchasing.cityofno.com/bso/external/advsearch/' + 'searchContract.sdo') req.add_header('Connection', 'keep-alive') req.add_header('DNT', '1') response = urllib2.urlopen(req) output = response.read() response.close() return output if __name__ == '__main__': log.info("Checking the city's purchasing site for new contracts") CheckCity().check_pages()
'external/advsearch/searchContract.sdo' req = urllib2.Request(url=url, data=data) req.add_header('Pragma', ' no-cache') req.add_header('Origin', 'http://www.purchasing.cityofno.com') req.add_header('Accept-Encoding', 'gzip, deflate') req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.add_header( 'Accept', 'text/add_contracthtml,application/xhtml+xml,application/xml;' + 'q=0.9,image/webp,*/*;q=0.8' ) req.add_header('Cache-Control', 'no-cache') req.add_header( 'Referer', 'http://www.purchasing.cityofno.com/bso/external/advsearch/' + 'searchContract.sdo' ) req.add_header('Connection', 'keep-alive') req.add_header('DNT', '1') response = urllib2.urlopen(req) output = response.read() response.close() return output if __name__ == '__main__': log.info("Checking the city's purchasing site for new contracts") CheckCity().check_pages()