Exemplo n.º 1
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with unlink request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.write:
         self.send_error(403)
         return
         
     self._update_policy(record, request)
     
     for removed in request['meta']['removed']:
         if removed in record['meta']:
             del record['meta'][removed]
     record['meta'].update(request['meta']['new'])
     
     database.update_record(record)
Exemplo n.º 2
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with unlink request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.write:
         self.send_error(403)
         return
     
     fs = state.get_filesystem(record['physical']['family'])
     try:
         fs.unlink(record)
     except filesystem.FileNotFoundError as e:
         _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
          'uid': uid,
         })
         self.send_error(404)
         return
     else:
         database.drop_record(uid)
Exemplo n.º 3
0
def main():
    # get the data from Craigslist
    housing = CraigslistHousing(site='sfbay',
                                area='sfc',
                                category='apa',
                                filters={
                                    'posted_today': True,
                                    'min_price': settings.min_price,
                                    'max_price': settings.max_price,
                                    'min_bedrooms': settings.min_bedrooms
                                })

    log.info('Retrieving listings')
    for result in housing.get_results(sort_by='newest', geotagged=True):
        # result = {'id': '6902060582', 'repost_of': None, 'name': 'Spacious one bedroom apartment near USF& GG PK', 'url': 'https://sfbay.craigslist.org/sfc/apa/d/san-francisco-spacious-one-bedroom/6902060582.html', 'datetime': '2019-05-31 21:44', 'price': '$2950', 'where': 'inner richmond', 'has_image': True, 'has_map': True, 'geotag': (37.775905, -122.458591), 'bedrooms': '1', 'area': None}

        # create a `listing` dict with the fields I care about and process the result
        listing = {}
        listing['craigslist_id'] = result['id']
        listing['craigslist_url'] = result['url']
        listing['posted_on'] = result['datetime']
        listing['description'] = result['name']
        listing['price'] = int(
            result['price'][1:]
        )  # price always has a leading '$' so we need to strip the leading character
        listing['neighborhood'] = str.lower(
            result['where']
        ) if result['where'] else ''  # sometimes this is null
        listing['num_bedrooms'] = result['bedrooms']
        listing['sqft'] = result['area']
        listing['latitude'] = result['geotag'][0]
        listing['longitude'] = result['geotag'][1]

        # decide if we want to notify about this listing
        # https://stackoverflow.com/questions/2783969/compare-string-with-all-values-in-array
        if any(x in listing['neighborhood']
               for x in settings.neighborhood_blacklist):
            notify = False
        else:
            notify = True

        # check if the listing is a duplicate
        if database.get_record(listing['craigslist_id']):
            log.info('Found duplicate record with ID {}, skipping'.format(
                listing['craigslist_id']))
            continue  # if duplicate we assume we've procsessed this listing so just skip it
        # otherwise we should save the listing and notify if applicable
        else:
            log.info('{} looks like a new listing, processing'.format(
                'craigslist_id'))

            # get the map image from Mapbox
            # we do this here instead of above to limit the number of API requests made to Mapbox
            listing['map_image'] = get_map(listing['latitude'],
                                           listing['longitude'])

            database.insert_record(listing)
            if notify is True:
                send_notification(listing)
                database.mark_as_notified(listing['craigslist_id'])
Exemplo n.º 4
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with retrieval request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.read:
         self.send_error(403)
         return
         
     current_time = int(time.time())
     record['physical']['atime'] = current_time
     record['stats']['accesses'] += 1
     for policy in ('delete', 'compress'):
         if 'stale' in record['policy'][policy]:
             record['policy'][policy]['staleTime'] = current_time + record['policy'][policy]['stale']
     database.update_record(record)
     
     fs = state.get_filesystem(record['physical']['family'])
     try:
         data = fs.get(record)
     except filesystem.FileNotFoundError as e:
         _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
          'uid': uid,
         })
         self.send_error(404)
         return
     else:
         _logger.debug("Evaluating decompression requirements...")
         applied_compression = record['physical']['format'].get('comp')
         supported_compressions = (c.strip() for c in (self.request.headers.get('Media-Storage-Supported-Compression') or '').split(';'))
         if applied_compression and not applied_compression in supported_compressions: #Must be decompressed first
             data = compression.get_decompressor(applied_compression)(data)
             applied_compression = None
             
         _logger.debug("Returning entity...")
         self.set_header('Content-Type', record['physical']['format']['mime'])
         if applied_compression:
             self.set_header('Media-Storage-Applied-Compression', applied_compression)
         while True:
             chunk = data.read(_CHUNK_SIZE)
             if chunk:
                 self.write(chunk)
             else:
                 break
Exemplo n.º 5
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with description request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.read:
         self.send_error(403)
         return
         
     _logger.debug("Describing entity...")
     record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record)
     record['uid'] = record['_id']
     del record['_id']
     del record['physical']['minRes']
     del record['keys']
     return record
Exemplo n.º 6
0
 def gets_url(original_id):
     return database.get_record(original_id)