def get_output_path(collection, package_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if collection == "BILLS":
# Store with the other bill data ([congress]/bills/[billtype]/[billtype][billnumber]).
bill_and_ver = get_bill_id_for_package(package_name, with_version=False, restrict_to_congress=options.get("congress"))
if not bill_and_ver:
return None # congress number does not match options["congress"]
from bills import output_for_bill
bill_id, version_code = bill_and_ver
return output_for_bill(bill_id, "text-versions/" + version_code, is_data_dot=False)
elif collection == "CRPT":
# Store committee reports in [congress]/crpt/[reporttype].
m = re.match(r"(\d+)([hse]rpt)(\d+)$", package_name)
if not m:
raise ValueError(package_name)
congress, report_type, report_number = m.groups()
if options.get("congress") and congress != options.get("congress"):
return None # congress number does not match options["congress"]
return "%s/%s/%s/%s/%s" % (utils.data_dir(), congress, collection.lower(), report_type, report_type + report_number)
else:
# Store in govinfo/COLLECTION/PKGNAME.
path = "%s/govinfo/%s/%s" % (utils.data_dir(), collection, package_name)
return path
def _checkValidationFile(self,path):
result = False
#copy the file and open it
self.xbmc_vfs.put(path + "xbmcbackup.val",xbmc.translatePath(utils.data_dir() + "xbmcbackup_restore.val"))
vFile = xbmcvfs.File(xbmc.translatePath(utils.data_dir() + "xbmcbackup_restore.val"),'r')
jsonString = vFile.read()
vFile.close()
#delete after checking
xbmcvfs.delete(xbmc.translatePath(utils.data_dir() + "xbmcbackup_restore.val"))
try:
json_dict = json.loads(jsonString)
if(xbmc.getInfoLabel('System.BuildVersion') == json_dict['xbmc_version']):
result = True
else:
result = xbmcgui.Dialog().yesno(utils.getString(30085),utils.getString(30086),utils.getString(30044))
except ValueError:
#may fail on older archives
result = True
return result
def get_output_path(collection, package_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if collection == "BILLS":
# Store with the other bill data ([congress]/bills/[billtype]/[billtype][billnumber]).
bill_and_ver = get_bill_id_for_package(
package_name,
with_version=False,
restrict_to_congress=options.get("congress"))
if not bill_and_ver:
return None # congress number does not match options["congress"]
from bills import output_for_bill
bill_id, version_code = bill_and_ver
return output_for_bill(bill_id,
"text-versions/" + version_code,
is_data_dot=False)
elif collection == "CRPT":
# Store committee reports in [congress]/crpt/[reporttype].
m = re.match(r"(\d+)([hse]rpt)(\d+)$", package_name)
if not m:
raise ValueError(package_name)
congress, report_type, report_number = m.groups()
if options.get("congress") and congress != options.get("congress"):
return None # congress number does not match options["congress"]
return "%s/%s/%s/%s/%s" % (utils.data_dir(), congress,
collection.lower(), report_type,
report_type + report_number)
else:
# Store in govinfo/COLLECTION/PKGNAME.
path = "%s/govinfo/%s/%s" % (utils.data_dir(), collection,
package_name)
return path
def setup(self):
#create authorization helper and load default settings
gauth = GoogleAuth(xbmc.validatePath(xbmc.translatePath(utils.addon_dir() + '/resources/lib/pydrive/settings.yaml')))
gauth.LoadClientConfigSettings()
#check if this user is already authorized
if(not xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "google_drive.dat"))):
settings = {"client_id":self.CLIENT_ID,'client_secret':self.CLIENT_SECRET}
drive_url = gauth.GetAuthUrl(settings)
utils.log("Google Drive Authorize URL: " + drive_url)
code = xbmcgui.Dialog().input('Google Drive Validation Code','Input the Validation code after authorizing this app')
gauth.Auth(code)
gauth.SaveCredentialsFile(xbmc.validatePath(xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))
else:
gauth.LoadCredentialsFile(xbmc.validatePath(xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))
#create the drive object
self.drive = GoogleDrive(gauth)
#make sure we have the folder we need
xbmc_folder = self._getGoogleFile(self.root_path)
print xbmc_folder
if(xbmc_folder == None):
self.mkdir(self.root_path)
def _readCronFile(self):
if(not os.path.exists(xbmc.translatePath(utils.data_dir()))):
os.makedirs(xbmc.translatePath(utils.data_dir()))
adv_jobs = []
try:
doc = xml.dom.minidom.parse(xbmc.translatePath(utils.data_dir() + "cron.xml"))
for node in doc.getElementsByTagName("job"):
tempJob = CronJob()
tempJob.name = str(node.getAttribute("name"))
tempJob.command = str(node.getAttribute("command"))
tempJob.expression = str(node.getAttribute("expression"))
tempJob.show_notification = str(node.getAttribute("show_notification"))
tempJob.id = len(adv_jobs)
utils.log(tempJob.name + " " + tempJob.expression + " loaded")
adv_jobs.append(tempJob)
except IOError:
#the file doesn't exist, return empty array
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("cron")
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(utils.data_dir() + "cron.xml"),"w")
doc.writexml(f," ")
f.close()
return adv_jobs
def _readCronFile(self):
if (not os.path.exists(xbmc.translatePath(utils.data_dir()))):
os.makedirs(xbmc.translatePath(utils.data_dir()))
adv_jobs = []
try:
doc = xml.dom.minidom.parse(
xbmc.translatePath(utils.data_dir() + "cron.xml"))
for node in doc.getElementsByTagName("job"):
tempJob = CronJob()
tempJob.name = str(node.getAttribute("name"))
tempJob.command = str(node.getAttribute("command"))
tempJob.expression = str(node.getAttribute("expression"))
tempJob.show_notification = str(
node.getAttribute("show_notification"))
tempJob.id = len(adv_jobs)
utils.log(tempJob.name + " " + tempJob.expression + " loaded",
xbmc.LOGDEBUG)
adv_jobs.append(tempJob)
except IOError:
#the file doesn't exist, return empty array
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("cron")
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(utils.data_dir() + "cron.xml"), "w")
doc.writexml(f, " ")
f.close()
return adv_jobs
def _createValidationFile(self):
vFile = xbmcvfs.File(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),'w')
vFile.write(json.dumps({"name":"XBMC Backup Validation File","xbmc_version":xbmc.getInfoLabel('System.BuildVersion')}))
vFile.write("")
vFile.close()
self.remote_vfs.put(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),self.remote_vfs.root_path + "xbmcbackup.val")
def __init__(self):
self.dataBasePath = os.path.join(xbmc.translatePath(utils.data_dir()), 'History.db')
#use scripts home for reading SQL files
self.sqlDir = os.path.join(xbmc.translatePath(utils.addon_dir()), 'resources', 'database')
#quick check to make sure data_dir exists
if(not xbmcvfs.exists(xbmc.translatePath(utils.data_dir()))):
xbmcvfs.mkdir(xbmc.translatePath(utils.data_dir()))
def getToken(self):
#get tokens, if they exist
if(xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "tokens.txt"))):
token_file = open(xbmc.translatePath(utils.data_dir() + "tokens.txt"))
key,secret = token_file.read().split('|')
token_file.close()
return [key,secret]
else:
return ["",""]
def _createValidationFile(self):
vFile = xbmcvfs.File(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),'w')
vFile.write(json.dumps({"name":"XBMC Backup Validation File","xbmc_version":xbmc.getInfoLabel('System.BuildVersion')}))
vFile.write("")
vFile.close()
success = self.remote_vfs.put(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),self.remote_vfs.root_path + "xbmcbackup.val")
#remove the validation file
xbmcvfs.delete(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"))
return success
def getToken(self):
#get tokens, if they exist
if (xbmcvfs.exists(xbmc.translatePath(utils.data_dir() +
"tokens.txt"))):
token_file = open(
xbmc.translatePath(utils.data_dir() + "tokens.txt"))
key, secret = token_file.read().split('|')
token_file.close()
return [key, secret]
else:
return ["", ""]
def get_output_path(sitemap, package_name, granule_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if sitemap["collection"] == "BILLS":
# Store with the other bill data.
bill_and_ver = get_bill_id_for_package(
package_name,
with_version=False,
restrict_to_congress=options.get("congress"))
if not bill_and_ver:
return None # congress number does not match options["congress"]
from bills import output_for_bill
bill_id, version_code = bill_and_ver
return output_for_bill(bill_id,
"text-versions/" + version_code,
is_data_dot=False)
else:
# Store in fdsys/COLLECTION/YEAR/PKGNAME[/GRANULE_NAME].
path = "%s/fdsys/%s/%s/%s" % (utils.data_dir(), sitemap["collection"],
sitemap["year"], package_name)
if granule_name:
path += "/" + granule_name
return path
def _writeHosts(self):
data_dir = utils.data_dir()
try:
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("hosts")
#create the child
for aHost in self.hosts:
newChild = doc.createElement("host")
newChild.setAttribute("name",str(aHost.name))
newChild.setAttribute("address",str(aHost.address))
newChild.setAttribute("port",str(aHost.port))
newChild.setAttribute("user",str(aHost.user))
newChild.setAttribute("password",str(aHost.password))
rootNode.appendChild(newChild)
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(data_dir + "hosts.xml"),'w')
doc.writexml(f," ")
f.close()
except IOError:
utils.log("Error writing hosts file")
def _readHosts(self):
data_dir = utils.data_dir()
if (not xbmcvfs.exists(data_dir)):
xbmcvfs.mkdir(data_dir)
try:
doc = xml.dom.minidom.parse(
xbmc.translatePath(data_dir + "hosts.xml"))
for node in doc.getElementsByTagName("host"):
self.hosts.append(
XbmcHost(str(node.getAttribute("name")),
str(node.getAttribute("address")),
int(node.getAttribute("port")),
str(node.getAttribute("user")),
str(node.getAttribute("password"))))
#sort the lists
self._sort()
except IOError:
#the file doesn't exist, create it
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("hosts")
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(data_dir + "hosts.xml"), 'w')
doc.writexml(f, " ")
f.close()
def _writeHosts(self):
data_dir = utils.data_dir()
try:
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("hosts")
#create the child
for aHost in self.hosts:
newChild = doc.createElement("host")
newChild.setAttribute("name", str(aHost.name))
newChild.setAttribute("address", str(aHost.address))
newChild.setAttribute("port", str(aHost.port))
newChild.setAttribute("user", str(aHost.user))
newChild.setAttribute("password", str(aHost.password))
rootNode.appendChild(newChild)
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(data_dir + "hosts.xml"), 'w')
doc.writexml(f, " ")
f.close()
except IOError:
utils.log("Error writing hosts file")
def _writeCronFile(self):
#write the cron file in full
try:
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("cron")
doc.appendChild(rootNode)
for aJob in self.jobs:
#create the child
newChild = doc.createElement("job")
newChild.setAttribute("name",aJob.name)
newChild.setAttribute("expression",aJob.expression)
newChild.setAttribute("command",aJob.command)
newChild.setAttribute("show_notification",aJob.show_notification)
rootNode.appendChild(newChild)
#write the file
f = open(xbmc.translatePath(utils.data_dir() + "cron.xml"),"w")
doc.writexml(f," ")
f.close()
except IOError:
self.log("error writing cron file")
def run(options):
# accepts yyyymmdd format
given_week = options.get('week_of', None)
if given_week is None:
for_the_week = get_latest_monday(options)
else:
for_the_week = get_monday_of_week(given_week)
if for_the_week is None:
return
logging.warn(
'Scraping upcoming bills from docs.house.gov/floor for the week of %s.\n'
% for_the_week)
house_floor = fetch_floor_week(for_the_week, options)
output_file = "%s/upcoming_house_floor/%s.json" % (utils.data_dir(),
for_the_week)
output = json.dumps(house_floor,
sort_keys=True,
indent=2,
default=utils.format_datetime)
utils.write(output, output_file)
logging.warn("\nFound %i bills for the week of %s, written to %s" %
(len(house_floor['upcoming']), for_the_week, output_file))
def _writeCronFile(self):
#write the cron file in full
try:
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("cron")
doc.appendChild(rootNode)
for aJob in self.jobs:
#create the child
newChild = doc.createElement("job")
newChild.setAttribute("name", aJob.name)
newChild.setAttribute("expression", aJob.expression)
newChild.setAttribute("command", aJob.command)
newChild.setAttribute("show_notification",
aJob.show_notification)
rootNode.appendChild(newChild)
#write the file
f = open(xbmc.translatePath(utils.data_dir() + "cron.xml"), "w")
doc.writexml(f, " ")
f.close()
except IOError:
utils.log("error writing cron file", xbmc.LOGERROR)
def output_for_bill(bill_id, format, is_data_dot=True):
bill_type, number, congress = utils.split_bill_id(bill_id)
if is_data_dot:
fn = "data.%s" % format
else:
fn = format
return "%s/%s/bills/%s/%s%s/%s" % (utils.data_dir(), congress, bill_type, bill_type, number, fn)
def run(options):
root_dir = utils.data_dir() + '/fdsys/STATUTE'
if "volume" in options:
to_fetch = glob.glob(root_dir + "/*/STATUTE-" +
str(int(options["volume"])))
elif "volumes" in options:
start, end = options["volumes"].split("-")
to_fetch = []
for v in xrange(int(start), int(end) + 1):
to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v)))
elif "year" in options:
to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) +
"/STATUTE-*")
elif "years" in options:
start, end = options["years"].split("-")
to_fetch = []
for y in xrange(int(start), int(end) + 1):
to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*"))
else:
to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*"))
logging.warn("Going to process %i volumes" % len(to_fetch))
utils.process_set(to_fetch, proc_statute_volume, options)
def run(options):
pages = []
if options.get('page'):
pages = [int(options.get('page'))]
elif options.get('pages'):
begin = int(options.get('begin', 1))
pages = list(range(begin, int(options.get('pages'))+1))
else:
pages = [1]
per_page = int(options.get('per_page', 200))
limit = options.get('limit')
count = 0
print("Processing pages %i through %i." % (pages[0], pages[-1]))
# go through each requested page
for page in pages:
print("[%i] Loading page." % page)
page_path = "%s/state/pages/%i/%i.json" % (utils.data_dir(), per_page, page)
page_data = json.load(open(page_path))
for result in page_data['Results']:
do_document(result, page, options)
count += 1
if limit and (count >= int(limit)):
break
if limit and (count >= int(limit)):
break
print("All done! Processed %i documents." % count)
def save_documents(package, event_id):
uploaded_documents = []
# find/create directory
folder = str(int(event_id) / 100)
output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (folder, event_id)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# loop through package and save documents
for name in package.namelist():
# for documents that are not xml
if ".xml" not in name:
try:
bytes = package.read(name)
except:
print "Did not save to disk: file %s" % (name)
continue
file_name = "%s/%s" % (output_dir, name)
# save document
logging.info("saved " + file_name)
with open(file_name, "wb") as document_file:
document_file.write(bytes)
# try to make a text version
text_doc = text_from_pdf(file_name)
if text_doc != None:
uploaded_documents.append(text_doc)
uploaded_documents.append(name)
return uploaded_documents
def run(options):
# Load the committee metadata from the congress-legislators repository and make a
# mapping from thomas_id and house_id to the committee dict. For each committee,
# replace the subcommittees list with a dict from thomas_id to the subcommittee.
utils.require_congress_legislators_repo()
committees = { }
for c in utils.yaml_load("congress-legislators/committees-current.yaml"):
committees[c["thomas_id"]] = c
if "house_committee_id" in c: committees[c["house_committee_id"] + "00"] = c
c["subcommittees"] = dict((s["thomas_id"], s) for s in c.get("subcommittees", []))
for chamber in ("house", "senate"):
# Load any existing meetings file so we can recycle GUIDs generated for Senate meetings.
existing_meetings = []
output_file = utils.data_dir() + "/committee_meetings_%s.json" % chamber
if os.path.exists(output_file):
existing_meetings = json.load(open(output_file))
# Scrape for meeting info.
if chamber == "senate":
meetings = fetch_senate_committee_meetings(existing_meetings, committees, options)
else:
meetings = fetch_house_committee_meetings(existing_meetings, committees, options)
# Write out.
utils.write(json.dumps(meetings, sort_keys=True, indent=2, default=utils.format_datetime),
output_file)
def output_for_bill(bill_id, format, is_data_dot=True):
bill_type, number, congress = utils.split_bill_id(bill_id)
if is_data_dot:
fn = "data.%s" % format
else:
fn = format
return "%s/%s/bills/%s/%s%s/%s" % (utils.data_dir(), congress, bill_type, bill_type, number, fn)
def save_documents(package, event_id):
uploaded_documents = []
# find/create directory
folder = str(int(event_id) / 100)
output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (
folder, event_id)
if not os.path.exists(output_dir): os.makedirs(output_dir)
# loop through package and save documents
for name in package.namelist():
# for documents that are not xml
if ".xml" not in name:
try:
bytes = package.read(name)
except:
print("Did not save to disk: file %s" % (name))
continue
file_name = "%s/%s" % (output_dir, name)
# save document
logging.info("saved " + file_name)
with open(file_name, 'wb') as document_file:
document_file.write(bytes)
# try to make a text version
text_doc = text_from_pdf(file_name)
if text_doc != None:
uploaded_documents.append(text_doc)
uploaded_documents.append(name)
return uploaded_documents
def _readHosts(self):
data_dir = utils.data_dir()
if(not xbmcvfs.exists(data_dir)):
xbmcvfs.mkdir(data_dir)
try:
doc = xml.dom.minidom.parse(xbmc.translatePath(data_dir + "hosts.xml"))
for node in doc.getElementsByTagName("host"):
self.hosts.append(XbmcHost(str(node.getAttribute("name")),str(node.getAttribute("address")),int(node.getAttribute("port"))))
#sort the lists
self._sort()
except IOError:
#the file doesn't exist, create it
doc = xml.dom.minidom.Document()
rootNode = doc.createElement("hosts")
doc.appendChild(rootNode)
#write the file
f = open(xbmc.translatePath(data_dir + "hosts.xml"),'w')
doc.writexml(f," ")
f.close()
def save_file(url, event_id):
# not saving xml but I cold be convinced otherwise
if ".xml" in url: return False
r = requests.get(url, stream=True)
if r.status_code == requests.codes.ok:
# find or create directory
folder = str(int(event_id) / 100)
output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (
folder, event_id)
if not os.path.exists(output_dir): os.makedirs(output_dir)
# get file name
splinter = url.split('/')
name = splinter[-1]
file_name = "%s/%s" % (output_dir, name)
# try to save
try:
logging.info("saved " + url + " to " + file_name)
with open(file_name, 'wb') as document_file:
document_file.write(r.content)
if ".pdf" in file_name:
text_doc = text_from_pdf(file_name)
return True
except:
print("Failed to save- %s" % (url))
return False
else:
logging.info("failed to fetch: " + url)
return False
def run(options):
# Load the committee metadata from the congress-legislators repository and make a
# mapping from thomas_id and house_id to the committee dict. For each committee,
# replace the subcommittees list with a dict from thomas_id to the subcommittee.
utils.require_congress_legislators_repo()
committees = {}
for c in utils.yaml_load("congress-legislators/committees-current.yaml"):
committees[c["thomas_id"]] = c
if "house_committee_id" in c:
committees[c["house_committee_id"] + "00"] = c
c["subcommittees"] = dict(
(s["thomas_id"], s) for s in c.get("subcommittees", []))
for chamber in ("house", "senate"):
# Load any existing meetings file so we can recycle GUIDs generated for Senate meetings.
existing_meetings = []
output_file = utils.data_dir(
) + "/committee_meetings_%s.json" % chamber
if os.path.exists(output_file):
existing_meetings = json.load(open(output_file))
# Scrape for meeting info.
if chamber == "senate":
meetings = fetch_senate_committee_meetings(existing_meetings,
committees, options)
else:
meetings = fetch_house_committee_meetings(existing_meetings,
committees, options)
# Write out.
utils.write(
json.dumps(meetings,
sort_keys=True,
indent=2,
default=utils.format_datetime), output_file)
def save_file(url, event_id):
# not saving xml but I cold be convinced otherwise
if ".xml" in url:
return False
r = requests.get(url, stream=True)
if r.status_code == requests.codes.ok:
# find or create directory
folder = str(int(event_id) / 100)
output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (folder, event_id)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# get file name
splinter = url.split("/")
name = splinter[-1]
file_name = "%s/%s" % (output_dir, name)
# try to save
try:
logging.info("saved " + url + " to " + file_name)
with open(file_name, "wb") as document_file:
document_file.write(r.content)
if ".pdf" in file_name:
text_doc = text_from_pdf(file_name)
return True
except:
print "Failed to save- %s" % (url)
return False
else:
logging.info("failed to fetch: " + url)
return False
def write_report(report):
data_path = "%s/%s/%s/report.json" % (report['inspector'], report['year'], report['report_id'])
utils.write(
utils.json_for(report),
"%s/%s" % (utils.data_dir(), data_path)
)
return data_path
def _refreshJobs(self):
#check if we should read in a new files list
stat_file = xbmcvfs.Stat(xbmc.translatePath(utils.data_dir() + "cron.xml"))
if(stat_file.st_mtime() > self.last_read):
utils.log("File update, loading new jobs")
#update the file
self.jobs = self._readCronFile();
self.last_read = time.time()
def get_data_path(*args):
# Utility function to generate a part of the path
# to data/{congress}/bills/{billtype}/{billtypenumber}
# given as many path elements as are provided. args
# is a list of zero or more of congress, billtype,
# and billtypenumber (in order).
args = list(args)
if len(args) > 0:
args.insert(1, "bills")
return os.path.join(utils.data_dir(), *args)
def get_data_path(*args):
# Utility function to generate a part of the path
# to data/{congress}/bills/{billtype}/{billtypenumber}
# given as many path elements as are provided. args
# is a list of zero or more of congress, billtype,
# and billtypenumber (in order).
args = list(args)
if len(args) > 0:
args.insert(1, "bills")
return os.path.join(utils.data_dir(), *args)
def download_report(report):
report_path = "%s/%s/%s/report.%s" % (report['inspector'], report['year'], report['report_id'], report['file_type'])
binary = (report['file_type'] == 'pdf')
utils.download(
report['url'],
"%s/%s" % (utils.data_dir(), report_path),
{'binary': binary}
)
return report_path
def output_for_vote(vote_id, format):
vote_chamber, vote_number, vote_congress, vote_session_year = utils.split_vote_id(vote_id)
return "%s/%s/votes/%s/%s%s/%s" % (
utils.data_dir(),
vote_congress,
vote_session_year,
vote_chamber,
vote_number,
"data.%s" % format,
)
def _refreshJobs(self):
#check if we should read in a new files list
stat_file = xbmcvfs.Stat(xbmc.translatePath(utils.data_dir() + "cron.xml"))
if(stat_file.st_mtime() > self.last_read):
utils.log("File update, loading new jobs",xbmc.LOGDEBUG)
#update the file
self.jobs = self._readCronFile();
self.last_read = time.time()
def mirror_bulkdata_file(sitemap, url, item_path, lastmod, options):
# Return a list of files we downloaded.
results = []
# Where should we store the file?
path = "%s/fdsys/%s/%s" % (utils.data_dir(), sitemap["collection"],
item_path)
# For BILLSTATUS, store this along with where we store the rest of bill
# status data.
if sitemap["collection"] == "BILLSTATUS":
from bills import output_for_bill
bill_id, version_code = get_bill_id_for_package(os.path.splitext(
os.path.basename(item_path))[0],
with_version=False)
path = output_for_bill(bill_id,
FDSYS_BILLSTATUS_FILENAME,
is_data_dot=False)
# Where should we store the lastmod found in the sitemap so that
# we can tell later if the file has changed?
lastmod_cache_file = os.path.splitext(path)[0] + "-lastmod.txt"
# Do we already have this file up to date?
if os.path.exists(lastmod_cache_file) and not options.get("force", False):
if lastmod == utils.read(lastmod_cache_file):
return
# With --cached, skip if the file is already downloaded.
if os.path.exists(path) and options.get("cached", False):
return
# Download.
logging.warn("Downloading: " + path)
data = utils.download(
url,
path,
utils.merge(
options,
{
'binary': True,
'force': True, # decision to cache was made above
'to_cache': False,
}))
results.append(path)
if not data:
# Something failed.
return
# Write the current last modified date back to disk so we know the next time whether
# we need to fetch the file again.
utils.write(lastmod, lastmod_cache_file)
return results
def setup(self):
#create authorization helper and load default settings
gauth = GoogleAuth(
xbmc.validatePath(
xbmc.translatePath(utils.addon_dir() +
'/resources/lib/pydrive/settings.yaml')))
gauth.LoadClientConfigSettings()
#check if this user is already authorized
if (not xbmcvfs.exists(
xbmc.translatePath(utils.data_dir() + "google_drive.dat"))):
settings = {
"client_id": self.CLIENT_ID,
'client_secret': self.CLIENT_SECRET
}
drive_url = gauth.GetAuthUrl(settings)
utils.log("Google Drive Authorize URL: " + drive_url)
code = xbmcgui.Dialog().input(
'Google Drive Validation Code',
'Input the Validation code after authorizing this app')
gauth.Auth(code)
gauth.SaveCredentialsFile(
xbmc.validatePath(
xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))
else:
gauth.LoadCredentialsFile(
xbmc.validatePath(
xbmc.translatePath(utils.data_dir() + 'google_drive.dat')))
#create the drive object
self.drive = GoogleDrive(gauth)
#make sure we have the folder we need
xbmc_folder = self._getGoogleFile(self.root_path)
print xbmc_folder
if (xbmc_folder == None):
self.mkdir(self.root_path)
def get_node_stats(self):
filepath = os.path.join(data_dir(), 'addrbook.json')
self.hosts = parse_address(filepath)
self._connect(NODE_STATS_TIMEOUT)
result = {
"current_node_num": self.online_num,
"online_nodes": self.online_node,
"timestamp": int(time.time()),
"time_lapse": self.time_lapse
}
self.restore()
return result
def run(options):
# accepts yyyymmdd format
for_the_week = get_monday_of_week(options.get('week_of', None))
logging.warn('Scraping upcoming bills from docs.house.gov/floor for the week of %s.\n' % for_the_week)
house_floor = fetch_floor_week(for_the_week, options)
output_file = "%s/upcoming_house_floor/%s.json" % (utils.data_dir(), for_the_week)
output = json.dumps(house_floor, sort_keys=True, indent=2, default=utils.format_datetime)
utils.write(output, output_file)
logging.warn("\nFound %i bills for the week of %s, written to %s" % (len(house_floor['upcoming_bills']), for_the_week, output_file))
def run(options):
for_the_week = get_monday_week(options.get('for_the_week', None)) #yyyymmdd
logging.info('Scraping upcoming bills from docs.house.gov/floor for the week %s.' % for_the_week)
# Parse the content into upcoming_bills
upcoming_bills = fetch_bills_week(for_the_week, options)
# Write the json to data folder
output_file = utils.data_dir() + "/upcoming_bills_%s.json" % for_the_week
utils.write(json.dumps(upcoming_bills, sort_keys=True, indent=2, default=utils.format_datetime), output_file)
def process_event(event_id=None, locale="en"):
"""Launcher that actually runs the event processing.
:param event_id: The event id to process. If None the latest event will
be downloaded and processed.
:type event_id: str
:param locale: The locale that will be used. Default to en.
:type locale: str
"""
population_path = os.path.join(data_dir(), "exposure", "IDN_mosaic", "popmap10_all.tif")
# Use cached data where available
# Whether we should always regenerate the products
force_flag = False
if "INASAFE_FORCE" in os.environ:
force_string = os.environ["INASAFE_FORCE"]
if str(force_string).capitalize() == "Y":
force_flag = True
# We always want to generate en products too so we manipulate the locale
# list and loop through them:
locale_list = [locale]
if "en" not in locale_list:
locale_list.append("en")
# Now generate the products
for locale in locale_list:
# Extract the event
# noinspection PyBroadException
try:
if os.path.exists(population_path):
shake_event = ShakeEvent(
event_id=event_id, locale=locale, force_flag=force_flag, population_raster_path=population_path
)
else:
shake_event = ShakeEvent(event_id=event_id, locale=locale, force_flag=force_flag)
except (BadZipfile, URLError):
# retry with force flag true
if os.path.exists(population_path):
shake_event = ShakeEvent(
event_id=event_id, locale=locale, force_flag=True, population_raster_path=population_path
)
else:
shake_event = ShakeEvent(event_id=event_id, locale=locale, force_flag=True)
except:
LOGGER.exception("An error occurred setting up the shake event.")
return
LOGGER.info("Event Id: %s", shake_event)
LOGGER.info("-------------------------------------------")
shake_event.render_map(force_flag)
def run_for_week(for_the_week, options):
logging.info('Scraping upcoming bills from docs.house.gov/floor for the week of %s...' % for_the_week)
house_floor = fetch_floor_week(for_the_week, options)
if house_floor is None:
logging.warn("Nothing posted for the week of %s" % for_the_week)
return
output_file = "%s/upcoming_house_floor/%s.json" % (utils.data_dir(), for_the_week)
output = json.dumps(house_floor, sort_keys=True, indent=2, default=utils.format_datetime)
utils.write(output, output_file)
logging.warn("Found %i bills for the week of %s, written to %s" % (len(house_floor['upcoming']), for_the_week, output_file))
def run_for_week(for_the_week, options):
logging.info('Scraping upcoming bills from docs.house.gov/floor for the week of %s...' % for_the_week)
house_floor = fetch_floor_week(for_the_week, options)
if house_floor is None:
logging.warn("Nothing posted for the week of %s" % for_the_week)
return
output_file = "%s/upcoming_house_floor/%s.json" % (utils.data_dir(), for_the_week)
output = json.dumps(house_floor, sort_keys=True, indent=2, default=utils.format_datetime)
utils.write(output, output_file)
logging.warn("Found %i bills for the week of %s, written to %s" % (len(house_floor['upcoming']), for_the_week, output_file))
def get_output_path(year, collection, package_name, granule_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if collection == "BILLS":
# Store with the other bill data.
bill_id, version_code = get_bill_id_for_package(package_name, with_version=False, restrict_to_congress=options.get("congress"))
if not bill_id: return None # congress number does not match options["congress"]
return output_for_bill(bill_id, "text-versions/" + version_code, is_data_dot=False)
else:
# Store in fdsys/COLLECTION/YEAR/PKGNAME[/GRANULE_NAME].
path = "%s/fdsys/%s/%s/%s" % (utils.data_dir(), collection, year, package_name)
if granule_name: path += "/" + granule_name
return path
def dsm_mockup_complete_db(postgres_service_url, s3_client) -> Tuple[str, str]:
utils.create_full_tables(url=postgres_service_url)
bucket_name = BUCKET_NAME
s3_client.create_bucket(bucket_name, delete_contents_if_exists=True)
file_1 = {
"project_id": "161b8782-b13e-5840-9ae2-e2250c231001",
"node_id": "ad9bda7f-1dc5-5480-ab22-5fef4fc53eac",
"filename": "outputController.dat",
}
f = utils.data_dir() / Path("outputController.dat")
object_name = "{project_id}/{node_id}/{filename}".format(**file_1)
s3_client.upload_file(bucket_name, object_name, f)
file_2 = {
"project_id": "161b8782-b13e-5840-9ae2-e2250c231001",
"node_id": "a3941ea0-37c4-5c1d-a7b3-01b5fd8a80c8",
"filename": "notebooks.zip",
}
f = utils.data_dir() / Path("notebooks.zip")
object_name = "{project_id}/{node_id}/{filename}".format(**file_2)
s3_client.upload_file(bucket_name, object_name, f)
yield (file_1, file_2)
utils.drop_all_tables(url=postgres_service_url)
def _readFile(self):
if(xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + "custom_paths.xml"))):
xmlFile = xml.dom.minidom.parse(xbmc.translatePath(utils.data_dir() + "custom_paths.xml"));
#video paths
if(len(xmlFile.getElementsByTagName("video")) > 0):
vNode = xmlFile.getElementsByTagName("video")[0]
for aNode in vNode.getElementsByTagName("path"):
self.jobs.append(self._createSchedule(aNode, "video"))
#music paths
if(len(xmlFile.getElementsByTagName("music")) > 0):
mNode = xmlFile.getElementsByTagName("music")[0]
for aNode in mNode.getElementsByTagName("path"):
self.jobs.append(self._createSchedule(aNode, "music"))
utils.log("Found " + str(len(self.jobs)) + " custom paths",xbmc.LOGDEBUG)
else:
utils.log("No custom file, skipping")
def get_config_path(self):
config_paths = [
join(data_dir(), ".pdrc_developer"), # dev config
expanduser("~/.pdrc"), # effective user homedir
]
try:
# logged in user (important for sudo), but not all distros support it
config_paths.append(join('/home', getlogin(), '.pdrc'))
except OSError:
pass
for path in config_paths:
if isfile(path):
return path
return None
def get_output_path(year, collection, package_name, granule_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if collection == "BILLS":
# Store with the other bill data.
m = re.match(r"BILLS-(\d+)([a-z]+)(\d+)(\D.*)", package_name)
if not m: raise Exception("Unmatched bill document package name: " + package_name)
congress, bill_type, bill_number, version_code = m.groups()
congress = int(congress)
if "congress" in options and congress != int(options["congress"]): return None
return output_for_bill(congress, bill_type, bill_number, "text-versions/" + version_code)
else:
# Store in fdsys/COLLECTION/YEAR/PKGNAME[/GRANULE_NAME].
path = "%s/fdsys/%s/%s/%s" % (utils.data_dir(), collection, year, package_name)
if granule_name: path += "/" + granule_name
return path
def mirror_bulkdata_file(collection, url, item_path, lastmod, options):
# Return a list of files we downloaded.
results = []
# Where should we store the file?
path = "%s/govinfo/%s/%s" % (utils.data_dir(), collection, item_path)
# For BILLSTATUS, store this along with where we store the rest of bill
# status data.
if collection == "BILLSTATUS":
from bills import output_for_bill
bill_id, version_code = get_bill_id_for_package(os.path.splitext(os.path.basename(item_path.replace("BILLSTATUS-", "")))[0], with_version=False)
path = output_for_bill(bill_id, FDSYS_BILLSTATUS_FILENAME, is_data_dot=False)
# Where should we store the lastmod found in the sitemap so that
# we can tell later if the file has changed?
lastmod_cache_file = os.path.splitext(path)[0] + "-lastmod.txt"
# Do we already have this file up to date?
if os.path.exists(lastmod_cache_file) and not options.get("force", False):
if lastmod == utils.read(lastmod_cache_file):
return
# With --cached, skip if the file is already downloaded.
if os.path.exists(path) and options.get("cached", False):
return
# Download.
logging.warn("Downloading: " + path)
data = utils.download(url, path, utils.merge(options, {
'binary': True,
'force': True, # decision to cache was made above
'to_cache': False,
}))
results.append(path)
if not data:
# Something failed.
return
# Write the current last modified date back to disk so we know the next time whether
# we need to fetch the file again.
utils.write(lastmod, lastmod_cache_file)
return results
def get_output_path(year, collection, package_name, granule_name, options):
# Where to store the document files?
# The path will depend a bit on the collection.
if collection == "BILLS":
# Store with the other bill data.
m = re.match(r"BILLS-(\d+)([a-z]+)(\d+)(\D.*)", package_name)
if not m:
raise Exception("Unmatched bill document package name: " +
package_name)
congress, bill_type, bill_number, version_code = m.groups()
congress = int(congress)
if "congress" in options and congress != int(options["congress"]):
return None
return output_for_bill(congress, bill_type, bill_number,
"text-versions/" + version_code)
else:
# Store in fdsys/COLLECTION/YEAR/PKGNAME[/GRANULE_NAME].
path = "%s/fdsys/%s/%s/%s" % (utils.data_dir(), collection, year,
package_name)
if granule_name: path += "/" + granule_name
return path
def run_data(options):
# for now, default to records
doc_type = options.get("type", "record")
agency = options.get("agency")
year = options.get("year")
doc_id = options.get("id")
if agency and year and doc_id:
get_record(agency, year, doc_id, options)
else:
# e.g. data/foiaonline/data/record/EPA/2014/090004d2803333d6/
if not doc_type: doc_type = "*"
if not year: year = "*"
# let people use lower-case slugs for agency on CLI
if agency:
agency = agency.upper()
else:
agency = "*"
doc_paths = glob.glob(os.path.join(
utils.data_dir(),
"foiaonline/meta/%s/%s/%s/*" % (doc_type, agency, year))
)
doc_paths.sort()
logging.warn("Going to fetch %i records." % len(doc_paths))
for doc_path in doc_paths:
pieces = doc_path.split("/")
this_doc_type = pieces[-4]
this_agency = pieces[-3]
this_year = pieces[-2]
this_doc_id = os.path.splitext(pieces[-1])[0]
if this_doc_type == "record":
get_record(this_agency, this_year, this_doc_id, options)
def run_data(options):
# for now, default to records
doc_type = options.get("type", "record")
agency = options.get("agency")
year = options.get("year")
doc_id = options.get("id")
if agency and year and doc_id:
get_record(agency, year, doc_id, options)
else:
# e.g. data/foiaonline/data/record/EPA/2014/090004d2803333d6/
if not doc_type: doc_type = "*"
if not year: year = "*"
# let people use lower-case slugs for agency on CLI
if agency:
agency = agency.upper()
else:
agency = "*"
doc_paths = glob.glob(
os.path.join(
utils.data_dir(),
"foiaonline/meta/%s/%s/%s/*" % (doc_type, agency, year)))
doc_paths.sort()
logging.warn("Going to fetch %i records." % len(doc_paths))
for doc_path in doc_paths:
pieces = doc_path.split("/")
this_doc_type = pieces[-4]
this_agency = pieces[-3]
this_year = pieces[-2]
this_doc_id = os.path.splitext(pieces[-1])[0]
if this_doc_type == "record":
get_record(this_agency, this_year, this_doc_id, options)
def _createValidationFile(self):
vFile = xbmcvfs.File(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),'w')
vFile.write(json.dumps({"name":"XBMC Backup Validation File","xbmc_version":xbmc.getInfoLabel('System.BuildVersion')}))
vFile.write("")
vFile.close()
success = self.remote_vfs.put(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"),self.remote_vfs.root_path + "xbmcbackup.val")
#remove the validation file
xbmcvfs.delete(xbmc.translatePath(utils.data_dir() + "xbmcbackup.val"))
if(success):
#android requires a .nomedia file to not index the directory as media
if(not xbmcvfs.exists(xbmc.translatePath(utils.data_dir() + ".nomedia"))):
nmFile = xbmcvfs.File(xbmc.translatePath(utils.data_dir() + ".nomedia"),'w')
nmFile.close()
success = self.remote_vfs.put(xbmc.translatePath(utils.data_dir() + ".nomedia"),self.remote_vfs.root_path + ".nomedia")
return success
def output_for_bill_version(bill_version_id):
bill_type, number, congress, version_code = utils.split_bill_version_id(
bill_version_id)
return "%s/%s/bills/%s/%s%s/text-versions/%s/data.json" % (
utils.data_dir(), congress, bill_type, bill_type, number, version_code)
def output_for(chamber):
return utils.data_dir() + "/committee_meetings_%s.json" % chamber
def setToken(self, key, secret):
#write the token files
token_file = open(xbmc.translatePath(utils.data_dir() + "tokens.txt"),
'w')
token_file.write("%s|%s" % (key, secret))
token_file.close()
def deleteToken(self):
if (xbmcvfs.exists(xbmc.translatePath(utils.data_dir() +
"tokens.txt"))):
xbmcvfs.delete(xbmc.translatePath(utils.data_dir() + "tokens.txt"))
