def upload(self, local_file, remote_file, verbose=True): """ Upload a file into the current bucket @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded/float(total_size))*100 sys.stdout.write( "\r-> %d / %d (%d%s) %s" % (uploaded, total_size, pct, "%", c) ) sys.stdout.flush() dst_uri = boto.storage_uri(self._bucket_name + "/" + remote_file, Bucket.STORAGE) if not verbose: callback = None f = file(local_file, "r") key = self._bucket.new_key(remote_file) key.set_contents_from_file(f, cb=callback) f.close() print "" # Update the key info key = self._bucket.get_key(remote_file) modified_date = utils.parse_iso_date(key.last_modified) return [modified_date, key.size, key.etag]
def __init__(self, api_object): self.date = parse_iso_date(api_object['dated_on']) self.amount = Decimal(api_object['amount']) self.unexplained_amount = Decimal(api_object['unexplained_amount']) self.full_description = api_object['full_description'] self.is_manual = api_object['is_manual'] self.url = api_object['url']
def ls(self): """ Perform an object listing for this bucket """ results = {} for key in self._bucket.list(): fdate = utils.parse_iso_date(key.last_modified) results[key.name] = [fdate, key.size, key.etag] return results
def generate_session_pages(self): self.date_data = get_date_dataset() self.date_data.reverse() if self.fast_run: COUNTER = 0 for leg, sess, num, d, dpub, page_start, page_end in self.date_data: dateobj = parse_iso_date(d) session = get_session_from_legsessnum(leg, sess, num) if not session: log.warn("File for %s-%s-%s is missing from the transcripts dataset!" % (leg, sess, num)) continue target_dir = "%s%d/%02d/%02d" % (self.sessions_path, dateobj.year, dateobj.month, dateobj.day) filename = "%s/index.html" % target_dir info = get_session_info(leg, sess, num) create_dir(os.path.join(self.output_dir, target_dir)) if type(session) in (str, unicode): # sessão em texto simples context = {'date': dateobj, 'year_number': dateobj.year, 'leg': leg, 'sess': sess, 'num': num, 'text': session, 'monthnames': MESES, 'pdf_url': 'xpto', 'page_name': 'sessoes', } if info: context['session_info'] = info self.render_template_into_file('session_plaintext.html', filename, context) elif type(session) in (dict, OrderedDict): # usar entradas do .json como contexto session['date'] = parse_iso_date(session['session_date']) session['monthnames'] = MESES session['page_name'] = 'sessoes' self.render_template_into_file('session.html', filename, session) if self.fast_run: COUNTER += 1 if COUNTER > self.fast_run_count: break
def get_file_properties(self, filename): """ @return tuple (filename, timestamp, filesize) """ d = self._cfg["files"].get(filename) # Parse the 'string' time-zone aware date. if d: d = copy.copy(d) d[0] = utils.parse_iso_date(d[0]); return d else: return None
def generate_datedict(fast_run=False): ''' Creates a dict with details for every day: * which weekday it is * whether there was a session on that day It's laid out in year - month - day - details order. Example: OrderedDict([(1976, {1: {1: {'weekday': 3, 'has_session': False}, 2: {'weekday': 4, 'has_session': False}, 3: {'weekday': 5, 'has_session': False}, ..... ''' # process dates into a year->dates dict datedict = OrderedDict() all_dates = [parse_iso_date(row[3]) for row in datedata] if fast_run: this_year = datetime.date.today().year all_years = [this_year, this_year - 1] else: all_years = list(set([d.year for d in all_dates])) for year in all_years: # populate it with its months # if current year, trim future months if int(year) == datetime.date.today().year: month = datetime.date.today().month months = range(1, month + 1) else: months = range(1, 13) datedict[year] = {} for month in months: datedict[year][month] = {} import calendar days_in_month = calendar.monthrange(year, month)[-1] all_days = range(1, days_in_month + 1) session_days = [datetime.date(d.year, d.month, d.day) for d in all_dates if d.month == month and d.year == year] for day_number in all_days: day_date = datetime.date(year, month, day_number) if day_date in session_days: has_session = True else: has_session = False datedict[year][month][day_number] = {'weekday': day_date.weekday(), 'has_session': has_session} if has_session: filename = get_session_filename_from_date(day_date) if filename and filename.endswith('json'): # sacar a topword s = json.load(open(filename, 'r')) if 'stats' in s: topword = s['stats']['topwords']['session'][0][0] datedict[year][month][day_number]['topword'] = topword return datedict
def generate_mp_pages(self): self.gov_data = get_gov_dataset() self.govpost_data = list(get_govpost_dataset()) self.gov_mp_ids = [int(row[2]) for row in self.govpost_data if row[2]] if not self.mps: self.mps = generate_mp_list(only_active=False) for mp in self.mps: id = int(mp['id']) mp['photo_url'] = self.photos_base_url + str(id) + ".jpg" # determine government posts if id in self.gov_mp_ids: mp['govposts'] = [] govpost_rows = [row for row in self.govpost_data if row[2].strip() and int(row[2]) == id] for row in govpost_rows: gov_number = int(row[0]) gov = None for r in self.gov_data: if int(r[1]) == gov_number: gov = {'number': r[1], 'start_date': parse_iso_date(r[2]), 'end_date': parse_iso_date(r[3]) if r[3] else None, } break if not gov: print(row) log.critical("Gov not found!") mp['govposts'].append({ 'post': row[3], 'start_date': parse_iso_date(row[4]), 'end_date': parse_iso_date(row[5]) if row[5] else None, 'gov': gov, }) # Parse dates for m in mp['mandates']: m['start_date'] = parse_iso_date(m['start_date']) if m.get('end_date'): m['end_date'] = parse_iso_date(m['end_date']) else: m['end_date'] = datetime.date.today() if mp.get('birthdate'): birthdate = parse_iso_date(mp['birthdate']) age = years_since(birthdate) else: age = None context = {'mp': mp, 'mp_age': age, 'l': None, 'page_name': 'deputados'} filename = os.path.join(self.mps_path, mp['slug'], 'index.html') self.render_template_into_file('mp_detail.html', filename, context)
def parse_freeagent_expenses(expenses_string): receipts = set() for expense in json.loads(expenses_string)['expenses']: try: filename = expense['attachment']['file_name'] except KeyError: filename = None receipts.add( Receipt( description=expense['description'], date=parse_iso_date(expense['dated_on']), amount=0 - Decimal(expense['gross_value']), filename=filename, )) return receipts
def render_list_elements(elements): """ Renders a HTML unordered list inner content from annotation export results. Args: elements (array): Array of dictionaries with a title and date key. Returns: The HTML string output. """ output = "" for el in elements: utf_title = el["title"].encode('utf-8') output += "<li>{0} (Last updated at {1})</li>".format(utf_title, parse_iso_date(el["date"])) return output
def download(self, remote_file, local_file, set_timestamp=True, verbose=True): """ Download a file from the bucket to the given local file path, and @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded / float(total_size)) * 100 sys.stdout.write( "\r <- [%d KB] %d / %d (%d%s) %s" % (total_size / 1024, uploaded, total_size, pct, "%", c)) sys.stdout.flush() if not verbose: callback = None path, filen = os.path.split(local_file) if not os.path.exists(path) and len(path.strip()) > 0: os.makedirs(path) key = self._bucket.get_key(remote_file) key.get_contents_to_filename(local_file, cb=callback) print "" if set_timestamp: modified_date = utils.parse_iso_date(key.last_modified) utils.set_timestamp_on_file(local_file, modified_date) return [key.last_modified, key.size, key.md5] ###############################################################################################
def upload(self, local_file, remote_file, verbose=True): """ Upload a file into the current bucket @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded / float(total_size)) * 100 sys.stdout.write("\r-> %d / %d (%d%s) %s" % (uploaded, total_size, pct, "%", c)) sys.stdout.flush() dst_uri = boto.storage_uri(self._bucket_name + "/" + remote_file, Bucket.STORAGE) if not verbose: callback = None f = file(local_file, "r") key = self._bucket.new_key(remote_file) key.set_contents_from_file(f, cb=callback) f.close() print "" # Update the key info key = self._bucket.get_key(remote_file) modified_date = utils.parse_iso_date(key.last_modified) return [modified_date, key.size, key.etag]
def download(self, remote_file, local_file, set_timestamp=True, verbose=True): """ Download a file from the bucket to the given local file path, and @return some relevent file properties """ global i ctr = ["-", "\\", "|", "/"] i = 0 def callback(uploaded, total_size): global i i = (i + 1) % len(ctr) c = ctr[i] if total_size == 0: pct = 100 else: pct = (uploaded/float(total_size))*100 sys.stdout.write( "\r <- [%d KB] %d / %d (%d%s) %s" % (total_size/1024, uploaded, total_size, pct, "%", c) ) sys.stdout.flush() if not verbose: callback = None path, filen = os.path.split(local_file) if not os.path.exists(path) and len(path.strip()) > 0: os.makedirs(path); key = self._bucket.get_key(remote_file) key.get_contents_to_filename(local_file, cb=callback); print "" if set_timestamp: modified_date = utils.parse_iso_date(key.last_modified) utils.set_timestamp_on_file(local_file, modified_date) return [key.last_modified, key.size, key.md5] ###############################################################################################