def get_publications(self, response: Response) -> list: """ Obtain the patents. :param response: response :return list of patents """ patents = [] for row in response.xpath( "//div[contains(@class, 'field-collection-item-field-publications')]/div[@class='content']" ): title = row.xpath( "string(div[contains(@class, 'field-name-field-link')])").get( ) other = row.xpath( "string(div[contains(@class, 'field-name-field-date-and-other-info')])" ).get() link = row.xpath( "div[contains(@class, 'field-name-field-link')]//a/@href").get( ) vendor = row.xpath( "string(div[contains(@class, 'field-name-field-publication')])" ).get() patent = create_product() patent['asset']['type'] = 8 patent['ref'] = link patent['contact']['website'] = link patent['name'] = title patent['abs'] = other patent['tag'] = remove_empty_string_from_array([vendor]) return patents
def parse(self, response): self.log('Parse technology {}'.format(response.url), level=logging.INFO) name = response.url.split('/')[-1] with open(os.path.join(self.work_directory, name + '.html'), 'wb') as fo: fo.write(response.body) product = create_product() product['ref'] = response.url product['tag'] = remove_empty_string_from_array( self.add_keywords(response)) product['asset']['type'] = 3 product['addr'] = deepcopy(self.address) product['name'] = response.xpath("string(//h1)").get() meta = self.get_meta(response) contents = meta['abstract'].split('\n') if len(contents) > 0 and len(contents[0]) > 0: product['abs'] = contents[0] else: product['abs'] = name product['intro'] = '\n'.join(contents[1:]) del meta['abstract'] product['asset']['market'] = dictionary_to_markdown(meta) manager, product['contact'] = self.get_contact(response) product['contact']['website'] = response.url inventors = self.add_inventors(response) for index, user in enumerate(inventors): user['abs'] = 'Inventor of ' + product['name'] user['addr'] = product['addr'] user['tag'] = product['tag'] with open(os.path.join(self.work_directory, name + '.json'), 'w') as fo: json.dump({'product': product, 'inventors': inventors}, fo)
def get_patents(self, response: Response) -> list: """ Obtain the patents. :param response: response :return list of patents """ patents = [] for row in response.xpath( "//div[contains(@class, 'field-collection-item-field-ip-info')]/div[@class='content']" ): title = row.xpath( "string(div[contains(@class, 'field-name-field-ip-title')])" ).get() tag = row.xpath( "string(div[contains(@class, 'field-name-field-ip-type')])" ).get() link = row.xpath( "div[contains(@class, 'field-name-field-ip-number-pctwo') or contains(@class, 'field-name-field-ip-number-pat-pend')]//a/@href" ).get() patent = create_product() patent['asset']['type'] = 1 patent['ref'] = link if link is not None else '' patent['contact']['website'] = link if link is not None else '' patent['name'] = title patent['tag'] = remove_empty_string_from_array([tag]) return patents
def get_contact(self, response: Response) -> (dict, dict): """ Gets the contact information. :param response: the response object :return: a tuple of two dict, one for an user and the other for the contact information """ contact = { 'email': '', 'phone': '', 'website': response.url, 'meet': '' } # manager name = response.xpath("//dd[@class='manager']/a/text()").get() link = response.xpath("//dd[@class='manager']/a/@href").get() manager = create_user() manager['name'] = name manager['ref'] = link tag = response.xpath("//dd[@class='manager']/div/em[1]/text()").get() if tag is not None and isinstance(tag, str): manager['tag'] = remove_empty_string_from_array(tag.split(', ')) contact['phone'] = response.xpath( "//dd[@class='manager']/div/em[2]/text()").get() manager['contact'] = contact manager['contact']['website'] = link self.log('find manager {} with contact {}'.format(manager, contact), level=logging.DEBUG) return manager, contact
def parse_device(cells): review_category = { '1': 'Ophthalmology and otorhinolaryngology', '2': 'dentistry', '3': 'cerebral, cardiovascular, respiratory, psychiatric, and neurological field', '3-1': 'Intervention devices mainly in cerebral, cardiovascular, respiratory, psychiatric, and neurological field', '3-2': 'Non-intervention devices mainly in cerebral, cardiovascular, respiratory, psychiatric, and neurological field', '4': 'cerebral, cardiovascular, respiratory, psychiatric, and neurological field', '5': 'gastrointestinal and urinary systems, obstetrics and gynecology', '6': 'orthopedic/plastic surgery and dermatology', '7': 'laboratory tests, in vitro diagnostics', '8': 'multicategory medical devices, advanced electronic medical devices, and other uncategorized medical devices', } p = create_product() p['name'] = cells[4] if isinstance(cells[3], datetime.datetime): p['created'] = cells[3].strftime("%a, %d %b %Y %H:%M:%S GMT") else: try: p['created'] = parser.parse( cells[3]).strftime("%a, %d %b %Y %H:%M:%S GMT") except: pass if isinstance(cells[1], datetime.datetime): p['updated'] = cells[1].strftime("%a, %d %b %Y %H:%M:%S GMT") else: try: p['updated'] = parser.parse( cells[1]).strftime("%a, %d %b %Y %H:%M:%S GMT") except: pass p['tag'] = remove_empty_string_from_array( [cells[5], cells[6], 'Japan PMDA', 'Medical Device']) p['asset']['lic'] = p['tag'] p['asset']['stat'] = 2 p['abs'] = review_category.get(cells[10], cells[10]) if len(p['abs']) < 1: p['abs'] = p['name'] p['asset']['market'] = cells[9] p['addr']['country'] = 'Japan' p['addr']['city'] = 'Unknown' a = create_company() a['name'] = cells[0] a['abs'] = 'A Medical Device Company' a['addr'] = p['addr'] a['tag'] = p['tag'] return p, a
def add_tags(self, response: Response) -> list: """ Add keywords to the project. :param response: Response object :return a list of inventors """ return remove_empty_string_from_array( response.xpath("//div[@id='categoryLinks']/div/a/text()").getall())
def add_tags(self, response: Response) -> list: """ Add keywords to the project. :param response: Response object :return a list of inventors """ return remove_empty_string_from_array( response.xpath( "//span[contains(@class, 'label')]/a/text()").getall())
def add_tags(self, response: Response) -> list: """ Add keywords to the project. :param response: Response object :return a list of inventors """ return remove_empty_string_from_array( response.xpath( '//*[@id="formTechPub1"]/div/table/tr/td[4]/div[1]/table/tr/td/a/text()' ).getall())
def add_tags(self, response: Response) -> list: """ Add keywords to the project. :param response: Response object :return a list of inventors """ return remove_empty_string_from_array([ remove_head_tail_white_space(t) for t in response.xpath( "//ul[@class='tech-category-list']//li/a/text()").getall() ])
def add_tags(self, response: Response) -> list: """ Add keywords to the project. :param response: Response object :return a list of inventors """ return remove_empty_string_from_array([ remove_head_tail_white_space(t) for t in response.xpath( "//div[@class='bdp']/p[contains(text(), 'Categories')]/text()" ).get().split(':')[-1].split('|') ])
def patent_callback(data: List[str], result: dict) -> None: if len(data) != len(self.PATENT_HEADER): self.logger.error('fail to parse {}'.format(data)) return d = {k: v for k, v in zip(self.PATENT_HEADER, data)} patent = create_product() patent['tag'].append(d['type']) patent['ref'] = d['number'] patent['addr']['country'] = d['country'] patent['updated'] = format_datetime(d['date']) patent['abs'] = d['abstract'] patent['name'] = d['title'] patent['tag'].append(d['kind']) patent['tag'] = remove_empty_string_from_array(patent['tag']) result[patent['ref']] = patent
def parse_drug(cells): review_category = { '1': 'Gastrointestinal drugs, dermatologic drugs, immunosuppressive drugs, and others (not classified as other categories)', '2': "Cardiovascular drugs, antiparkinsonian drugs, anti-Alzheimer's drugs", '3-1': 'Central/peripheral nervous system drugs (excluding anesthetic drugs)', '3-2': 'Anesthetic drugs, sensory organ drugs (excluding drugs for inflammatory diseases), narcotics', '4': 'Antibacterial drugs, antiviral drugs (excluding AIDS drugs), antifungal drugs, antiprotozoal drugs, anthelmintic drugs', '5': 'Reproductive system drugs, drugs for urogenital system, combination drugs', '6-1': 'Respiratory tract drugs, anti-allergy drugs (excluding dermatologic drugs), sensory organ drugs (drugs for inflammatory diseases)', '6-2': 'Hormone drugs, drugs for metabolic disorders (including diabetes mellitus, osteoporosis, gout, and inborn errors of metabolism)', } p = create_product() p['name'] = cells[3] if isinstance(cells[2], datetime): p['created'] = cells[2] else: try: p['created'] = parser.parse(cells[2]) except: pass category = review_category.get(cells[6], cells[6]) p['tag'] = remove_empty_string_from_array([category, 'Japan PMDA', 'Drug']) p['asset']['lic'] = p['tag'] p['asset']['stat'] = 2 p['asset']['tech'] = cells[0] p['abs'] = cells[5] if len(p['abs']) < 1: p['abs'] = p['name'] p['addr']['country'] = 'Japan' p['addr']['city'] = 'Unknown' a = create_company() a['name'] = cells[0] a['abs'] = 'A Drug Company' a['addr'] = p['addr'] a['tag'] = p['tag'] return p, a
def main(): product_code = get_product_code() log = create_logger('510K') result = json.load(open(os.path.expanduser('~/work/fda/device-classification-0001-of-0001.json'), 'r')) log.critical(datetime.datetime.now()) for r in result['results']: p = create_product() p['name'] = r.get('device_name', r['openfda'].get('device_name', '')) p['ref'] = r.get('k_number', r['openfda'].get('k_number', '')) p['addr']['line1'] = r.get('address_1', r['openfda'].get('address_1', '')) p['addr']['line2'] = r.get('address_2', r['openfda'].get('address_2', '')) p['addr']['city'] = r.get('city', r['openfda'].get('city', '')) p['addr']['state'] = r.get('state', r['openfda'].get('state', '')) p['addr']['zip'] = r.get('zip_code', r['openfda'].get('zip_code', '')) p['addr']['country'] = r.get('country_code', r['openfda'].get('country_code', '')) p['intro'] = r.get('statement_or_summary', r['openfda'].get('statement_or_summary', '')) p['asset']['type'] = 0 p['tag'] = [ r.get('advisory_committee_description', r['openfda'].get('advisory_committee_description', '')), r.get('medical_specialty_description', r['openfda'].get('medical_specialty_description', '')), 'FDA', 'Medical Device', '510K'] # p['tag'] is used for tags readable to common users, p['lic'] is used for tags specified for product. p['asset']['lic'] = [ 'FDA', '510K', r.get('clearance_type', r['openfda'].get('clearance_type', '')), r.get('advisory_committee_description', r['openfda'].get('advisory_committee_description', '')), r['openfda'].get('medical_specialty_description', ''), r.get('product_code', r['openfda'].get('product_code', '')), r.get('regulation_number', r['openfda'].get('regulation_number', '')), r.get('decision_description', r['openfda'].get('decision_description', '')), ] p['asset']['lic'].extend(third_party(r.get('third_party_flag', r['openfda'].get('third_party_flag', '')))) if len(r.get('expedited_review_flag', r['openfda'].get('expedited_review_flag', ''))) > 0: p['asset']['lic'].append('Expedited Review') if r.get('submission_type_id', r['openfda'].get('submission_type_id', '')) not in {'1', '2'} and \ submission_type(r.get('submission_type_id', r['openfda'].get('submission_type_id', ''))) is not None: p['asset']['lic'].append( submission_type(r.get('submission_type_id', r['openfda'].get('submission_type_id', '')))) p['tag'].append(submission_type(r.get('submission_type_id', r['openfda'].get('submission_type_id', '')))) code = product_code.get(r.get('product_code', r['openfda'].get('product_code', '')), None) if code is not None: p['abs'] = code['device_name'] p['asset']['lic'].extend([ 'Class ' + code['device_class'], 'GMP Exempt' if code['gmp_exempt_flag'] == 'N' else 'GMP Required', ]) p['tag'].append('Class ' + code['device_class']) if code['implant_flag'] != 'N': p['asset']['lic'].append('Implant') p['tag'].append('Implant') if code['life_sustain_support_flag'] != 'N': p['asset']['lic'].append('Life Sustain Support') p['tag'].append('Life Sustain Support') else: p['abs'] = p['name'] p['asset']['stat'] = map_status(r.get('decision_code', r['openfda'].get('decision_code', ''))) try: p['created'] = parser.parse(r.get('date_received', r['openfda'].get('date_received', None))).strftime( "%a, %d %b %Y %H:%M:%S GMT") except: pass try: p['updated'] = parser.parse(r.get('decision_date', r['openfda'].get('decision_date', None))).strftime( "%a, %d %b %Y %H:%M:%S GMT") except: pass p['asset']['lic'] = remove_empty_string_from_array(p['asset']['lic']) p['tag'] = remove_empty_string_from_array(p['tag']) a = create_company() a['name'] = r.get('applicant', r['openfda'].get('applicant', '')) a['abs'] = 'A Medical Device Company' a['addr'] = p['addr'] a['tag'] = p['tag'] a['group']['parentId'] = '000000000000000000000000' # contact is just the name of contact response = add_record('entity', [p, a]) if response['_status'] != 'OK': log.error('fail to create record for {}'.format(p['name'])) continue applicant_product = create_relationship(response['_items'][1]['_id'], response['_items'][0]['_id']) applicant_product['type'] = 7 applicant_product['name'] = 'Applicant' applicant_product['abs'] = 'Applicant' response = add_record('relationship', [applicant_product]) if response['_status'] != 'OK': log.error('fail to create relationship for {}'.format(p['name'])) else: log.debug('added {} to the system'.format(p['name'])) log.critical(datetime.datetime.now())