def upload(self, local_file, remote_file, verbose=True): """ Upload a file into the current bucket @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded/float(total_size))*100 sys.stdout.write( "\r-> %d / %d (%d%s) %s" % (uploaded, total_size, pct, "%", c) ) sys.stdout.flush() dst_uri = boto.storage_uri(self._bucket_name + "/" + remote_file, Bucket.STORAGE) if not verbose: callback = None f = file(local_file, "r") key = self._bucket.new_key(remote_file) key.set_contents_from_file(f, cb=callback) f.close() print "" # Update the key info key = self._bucket.get_key(remote_file) modified_date = utils.parse_iso_date(key.last_modified) return [modified_date, key.size, key.etag]
def __init__(self, api_object):
self.date = parse_iso_date(api_object['dated_on'])
self.amount = Decimal(api_object['amount'])
self.unexplained_amount = Decimal(api_object['unexplained_amount'])
self.full_description = api_object['full_description']
self.is_manual = api_object['is_manual']
self.url = api_object['url']
def ls(self):
"""
Perform an object listing for this bucket
"""
results = {}
for key in self._bucket.list():
fdate = utils.parse_iso_date(key.last_modified)
results[key.name] = [fdate, key.size, key.etag]
return results
def ls(self):
"""
Perform an object listing for this bucket
"""
results = {}
for key in self._bucket.list():
fdate = utils.parse_iso_date(key.last_modified)
results[key.name] = [fdate, key.size, key.etag]
return results
def generate_session_pages(self):
self.date_data = get_date_dataset()
self.date_data.reverse()
if self.fast_run:
COUNTER = 0
for leg, sess, num, d, dpub, page_start, page_end in self.date_data:
dateobj = parse_iso_date(d)
session = get_session_from_legsessnum(leg, sess, num)
if not session:
log.warn("File for %s-%s-%s is missing from the transcripts dataset!" % (leg, sess, num))
continue
target_dir = "%s%d/%02d/%02d" % (self.sessions_path, dateobj.year, dateobj.month, dateobj.day)
filename = "%s/index.html" % target_dir
info = get_session_info(leg, sess, num)
create_dir(os.path.join(self.output_dir, target_dir))
if type(session) in (str, unicode):
# sessão em texto simples
context = {'date': dateobj,
'year_number': dateobj.year,
'leg': leg,
'sess': sess,
'num': num,
'text': session,
'monthnames': MESES,
'pdf_url': 'xpto',
'page_name': 'sessoes',
}
if info:
context['session_info'] = info
self.render_template_into_file('session_plaintext.html', filename, context)
elif type(session) in (dict, OrderedDict):
# usar entradas do .json como contexto
session['date'] = parse_iso_date(session['session_date'])
session['monthnames'] = MESES
session['page_name'] = 'sessoes'
self.render_template_into_file('session.html', filename, session)
if self.fast_run:
COUNTER += 1
if COUNTER > self.fast_run_count:
break
def get_file_properties(self, filename): """ @return tuple (filename, timestamp, filesize) """ d = self._cfg["files"].get(filename) # Parse the 'string' time-zone aware date. if d: d = copy.copy(d) d[0] = utils.parse_iso_date(d[0]); return d else: return None
def generate_datedict(fast_run=False):
'''
Creates a dict with details for every day:
* which weekday it is
* whether there was a session on that day
It's laid out in year - month - day - details order.
Example:
OrderedDict([(1976, {1: {1: {'weekday': 3, 'has_session': False}, 2: {'weekday': 4, 'has_session': False}, 3: {'weekday': 5, 'has_session': False}, .....
'''
# process dates into a year->dates dict
datedict = OrderedDict()
all_dates = [parse_iso_date(row[3]) for row in datedata]
if fast_run:
this_year = datetime.date.today().year
all_years = [this_year, this_year - 1]
else:
all_years = list(set([d.year for d in all_dates]))
for year in all_years:
# populate it with its months
# if current year, trim future months
if int(year) == datetime.date.today().year:
month = datetime.date.today().month
months = range(1, month + 1)
else:
months = range(1, 13)
datedict[year] = {}
for month in months:
datedict[year][month] = {}
import calendar
days_in_month = calendar.monthrange(year, month)[-1]
all_days = range(1, days_in_month + 1)
session_days = [datetime.date(d.year, d.month, d.day) for d in all_dates
if d.month == month and d.year == year]
for day_number in all_days:
day_date = datetime.date(year, month, day_number)
if day_date in session_days:
has_session = True
else:
has_session = False
datedict[year][month][day_number] = {'weekday': day_date.weekday(),
'has_session': has_session}
if has_session:
filename = get_session_filename_from_date(day_date)
if filename and filename.endswith('json'):
# sacar a topword
s = json.load(open(filename, 'r'))
if 'stats' in s:
topword = s['stats']['topwords']['session'][0][0]
datedict[year][month][day_number]['topword'] = topword
return datedict
def generate_mp_pages(self):
self.gov_data = get_gov_dataset()
self.govpost_data = list(get_govpost_dataset())
self.gov_mp_ids = [int(row[2]) for row in self.govpost_data if row[2]]
if not self.mps:
self.mps = generate_mp_list(only_active=False)
for mp in self.mps:
id = int(mp['id'])
mp['photo_url'] = self.photos_base_url + str(id) + ".jpg"
# determine government posts
if id in self.gov_mp_ids:
mp['govposts'] = []
govpost_rows = [row for row in self.govpost_data if row[2].strip() and int(row[2]) == id]
for row in govpost_rows:
gov_number = int(row[0])
gov = None
for r in self.gov_data:
if int(r[1]) == gov_number:
gov = {'number': r[1],
'start_date': parse_iso_date(r[2]),
'end_date': parse_iso_date(r[3]) if r[3] else None,
}
break
if not gov:
print(row)
log.critical("Gov not found!")
mp['govposts'].append({
'post': row[3],
'start_date': parse_iso_date(row[4]),
'end_date': parse_iso_date(row[5]) if row[5] else None,
'gov': gov,
})
# Parse dates
for m in mp['mandates']:
m['start_date'] = parse_iso_date(m['start_date'])
if m.get('end_date'):
m['end_date'] = parse_iso_date(m['end_date'])
else:
m['end_date'] = datetime.date.today()
if mp.get('birthdate'):
birthdate = parse_iso_date(mp['birthdate'])
age = years_since(birthdate)
else:
age = None
context = {'mp': mp,
'mp_age': age,
'l': None,
'page_name': 'deputados'}
filename = os.path.join(self.mps_path, mp['slug'], 'index.html')
self.render_template_into_file('mp_detail.html', filename, context)
def parse_freeagent_expenses(expenses_string):
receipts = set()
for expense in json.loads(expenses_string)['expenses']:
try:
filename = expense['attachment']['file_name']
except KeyError:
filename = None
receipts.add(
Receipt(
description=expense['description'],
date=parse_iso_date(expense['dated_on']),
amount=0 - Decimal(expense['gross_value']),
filename=filename,
))
return receipts
def render_list_elements(elements):
"""
Renders a HTML unordered list inner content from annotation export results.
Args:
elements (array): Array of dictionaries with a title and date key.
Returns:
The HTML string output.
"""
output = ""
for el in elements:
utf_title = el["title"].encode('utf-8')
output += "<li>{0} (Last updated at {1})</li>".format(utf_title, parse_iso_date(el["date"]))
return output
def download(self,
remote_file,
local_file,
set_timestamp=True,
verbose=True):
"""
Download a file from the bucket to the given local file path, and
@return some relevent file properties
"""
global i
ctr = ["-", "\\", "|", "/"]
i = 0
def callback(uploaded, total_size):
global i
i = (i + 1) % len(ctr)
c = ctr[i]
if total_size == 0:
pct = 100
else:
pct = (uploaded / float(total_size)) * 100
sys.stdout.write(
"\r <- [%d KB] %d / %d (%d%s) %s" %
(total_size / 1024, uploaded, total_size, pct, "%", c))
sys.stdout.flush()
if not verbose:
callback = None
path, filen = os.path.split(local_file)
if not os.path.exists(path) and len(path.strip()) > 0:
os.makedirs(path)
key = self._bucket.get_key(remote_file)
key.get_contents_to_filename(local_file, cb=callback)
print ""
if set_timestamp:
modified_date = utils.parse_iso_date(key.last_modified)
utils.set_timestamp_on_file(local_file, modified_date)
return [key.last_modified, key.size, key.md5]
###############################################################################################
def upload(self, local_file, remote_file, verbose=True):
"""
Upload a file into the current bucket
@return some relevent file properties
"""
global i
ctr = ["-", "\\", "|", "/"]
i = 0
def callback(uploaded, total_size):
global i
i = (i + 1) % len(ctr)
c = ctr[i]
if total_size == 0:
pct = 100
else:
pct = (uploaded / float(total_size)) * 100
sys.stdout.write("\r-> %d / %d (%d%s) %s" %
(uploaded, total_size, pct, "%", c))
sys.stdout.flush()
dst_uri = boto.storage_uri(self._bucket_name + "/" + remote_file,
Bucket.STORAGE)
if not verbose:
callback = None
f = file(local_file, "r")
key = self._bucket.new_key(remote_file)
key.set_contents_from_file(f, cb=callback)
f.close()
print ""
# Update the key info
key = self._bucket.get_key(remote_file)
modified_date = utils.parse_iso_date(key.last_modified)
return [modified_date, key.size, key.etag]
def download(self, remote_file, local_file, set_timestamp=True, verbose=True): """ Download a file from the bucket to the given local file path, and @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded/float(total_size))*100 sys.stdout.write( "\r <- [%d KB] %d / %d (%d%s) %s" % (total_size/1024, uploaded, total_size, pct, "%", c) ) sys.stdout.flush() if not verbose: callback = None path, filen = os.path.split(local_file) if not os.path.exists(path) and len(path.strip()) > 0: os.makedirs(path); key = self._bucket.get_key(remote_file) key.get_contents_to_filename(local_file, cb=callback); print "" if set_timestamp: modified_date = utils.parse_iso_date(key.last_modified) utils.set_timestamp_on_file(local_file, modified_date) return [key.last_modified, key.size, key.md5] ###############################################################################################
