class Counter(MongoModel):
"""
Counter for auto-increment
"""
id = fields.CharField()
reference_value = fields.MongoBaseField()
seq = fields.IntegerField()
@staticmethod
def getNextSequence(name):
try:
rt = Counter.objects.raw({'id': name})
rt.update({'$inc': {'seq': 1}})
return rt.first().seq
except DoesNotExist:
new_counter = Counter(id=name, seq=0).save()
return Counter.getNextSequence(name)
class Meta:
collection_name = 'counters'
final = True
class Facility(MongoModel):
'''
Class for Facility. (https://publicfiles.fcc.gov/developer/)
'''
id = fields.MongoBaseField(primary_key=True)
facilityId = fields.MongoBaseField()
frequency = fields.MongoBaseField()
callSign = fields.MongoBaseField()
activeInd = fields.MongoBaseField()
metadata = fields.MongoBaseField()
def fetchMetadata(self):
if not self.metadata:
self.metadata = self.fetch()
return self
def search(keyword):
'''
Search for facility return array of results
URL: https://publicfiles.fcc.gov/api/service/facility/search/WjLA.json
'''
entityUrl = "service/facility/search/{keyword}".format(keyword=keyword)
response = callApi(entityUrl)
return Facility.facilityList(
response['results']['globalSearchResults']['tvFacilityList'])
def facilityList(facilityList, fetchMetadata=False):
'''
Return Facility objects from facilityList JSON response.
:return: list of objects of class facility.
'''
all_facilities = []
for facility_obj in facilityList:
facility_obj['facilityId'] = facility_obj.pop('id', None)
facility_obj['id'] = hashlib.sha224(
facility_obj['facilityId'].encode('utf-8')).hexdigest()
all_facilities.append(Facility(**facility_obj))
if fetchMetadata:
facilities = [
facility.fetchMetadata() for facility in all_facilities
if facility
]
return facilities
def fetch(self, serviceType="tv"):
'''
Call the facility API and return a populated facility object. Throw error if not found or more than one found.
URL: https://publicfiles.fcc.gov/api/service/tv/facility/id/{id}.json
'''
entityUrl = "service/{serviceType}/facility/id/{entityID}".format(
serviceType=serviceType, entityID=self.facilityId)
results = callApi(entityUrl)['results']
if results['facility']:
metadata = results['facility']
else:
metadata = None
return metadata
def getAll(serviceType="tv", fetchMetaData=False, save=False):
'''
Fetch all available facilities for given serviceType
:return: list of Facilities
'''
entityUrl = "service/{serviceType}/facility/getall".format(
serviceType=serviceType)
responses = callApi(entityUrl)
facilityList = responses['results']['facilityList']
if type(facilityList) is list and len(facilityList) > 0:
facilities = Facility.facilityList(facilityList,
fetchMetadata=fetchMetaData)
if save:
facilities = Facility.objects.bulk_create(facilities,
retrieve=True)
return facilities
class Ingredient(Item):
level = fields.IntegerField(required=True, default=1)
weight = fields.IntegerField(required=True, default=1)
itype = fields.IntegerField(required=True, default=12)
value = fields.IntegerField(required=True, default=1)
effect = fields.MongoBaseField(required=True, blank=True)
class Gemstone(Item):
itype = fields.IntegerField(required=True, default=11)
effect = fields.CharField(required=True)
amount = fields.MongoBaseField(required=True)
class Action(EmbeddedMongoModel):
actionType = fields.BooleanField(required=True)
values = fields.MongoBaseField(required=True)
class Meta:
final = True
class PoliticalFile(MongoModel):
def __init__(self, fileObj):
'''
Initiate a political file
{
"file_id": "e84e063a-eb1e-bd96-3f32-db76d78c9f36",
"file_name": "NextGen 01-18-17 to 01-23-17",
"file_extension": "pdf",
"file_size": "181094",
"file_status": "com_cvt",
"file_folder_path": "Political Files/2017/Non-Candidate Issue Ads/NextGen Climate Action/01-18-17 to 01-23-17",
"folder_id": "57bab4e3-b517-9126-12cf-d9adf9d87cbc",
"file_manager_id": "751f01eb-a938-41db-93dd-b47b719c2889",
"create_ts": "2017-01-19T13:18:08-05:00",
"last_update_ts": "2017-01-19T13:18:10-05:00"
}
'''
id = fields.MongoBaseField()
downloaded = fields.MongoBaseField(default=False)
needs_ocr = fields.MongoBaseField(default=False)
status = fields.MongoBaseField(default='initialized')
isnab = fields.MongoBaseField(default=False)
def fetchAll(entityId):
'''
Fetch all political files for a given facility
https://publicfiles.fcc.gov/api/manager/search/key/Political%20File.json?entityId=1051
'''
entityUrl = "manager/search/key/Political File.json"
params = {"entityId": entityId}
folder_results = callApi(entityUrl, params)
if folder_results:
return [
PoliticalFile(id=file.pop('file_id', 'Missing'), metadata=file)
for file in folder_results['searchResult']['files']
]
else:
return None
def download(self):
'''
Download a political file.
https://publicfiles.fcc.gov/api/manager/download/d51881d5-fe65-e976-81ee-3e5a626aaee6/7a21d601-1c1c-41b4-b8f2-a22dca06cade.pdf
:param self:
:return:
'''
folder_id = self.metadata['folder_id']
file_manager_id = self.metadata['file_manager_id']
file_name = self.metadata['file_name']
if 'NAB' not in file_name:
entityUrl = 'manager/download/{folder_id}/{file_manager_id}.pdf'.format(
folder_id=folder_id, file_manager_id=file_manager_id)
print(downloadFile(entityUrl))
else:
self.isnab = True
def needOcr(self):
'''
Check for embedded text to decide if OCR is needed
:return:
'''
def doOcr(self):
'''
Execute the OCR pipeline
:return:
'''
def tabulaExtraction(self):
'''