def _post(self):
request = _get_json(self.request.body)
uid = request['uid']
_logger.info("Proceeding with unlink request for '%(uid)s'..." % {
'uid': uid,
})
record = database.get_record(uid)
if not record:
self.send_error(404)
return
trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
if not trust.write:
self.send_error(403)
return
self._update_policy(record, request)
for removed in request['meta']['removed']:
if removed in record['meta']:
del record['meta'][removed]
record['meta'].update(request['meta']['new'])
database.update_record(record)
def _post(self):
request = _get_json(self.request.body)
uid = request['uid']
_logger.info("Proceeding with unlink request for '%(uid)s'..." % {
'uid': uid,
})
record = database.get_record(uid)
if not record:
self.send_error(404)
return
trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
if not trust.write:
self.send_error(403)
return
fs = state.get_filesystem(record['physical']['family'])
try:
fs.unlink(record)
except filesystem.FileNotFoundError as e:
_logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
'uid': uid,
})
self.send_error(404)
return
else:
database.drop_record(uid)
def main():
# get the data from Craigslist
housing = CraigslistHousing(site='sfbay',
area='sfc',
category='apa',
filters={
'posted_today': True,
'min_price': settings.min_price,
'max_price': settings.max_price,
'min_bedrooms': settings.min_bedrooms
})
log.info('Retrieving listings')
for result in housing.get_results(sort_by='newest', geotagged=True):
# result = {'id': '6902060582', 'repost_of': None, 'name': 'Spacious one bedroom apartment near USF& GG PK', 'url': 'https://sfbay.craigslist.org/sfc/apa/d/san-francisco-spacious-one-bedroom/6902060582.html', 'datetime': '2019-05-31 21:44', 'price': '$2950', 'where': 'inner richmond', 'has_image': True, 'has_map': True, 'geotag': (37.775905, -122.458591), 'bedrooms': '1', 'area': None}
# create a `listing` dict with the fields I care about and process the result
listing = {}
listing['craigslist_id'] = result['id']
listing['craigslist_url'] = result['url']
listing['posted_on'] = result['datetime']
listing['description'] = result['name']
listing['price'] = int(
result['price'][1:]
) # price always has a leading '$' so we need to strip the leading character
listing['neighborhood'] = str.lower(
result['where']
) if result['where'] else '' # sometimes this is null
listing['num_bedrooms'] = result['bedrooms']
listing['sqft'] = result['area']
listing['latitude'] = result['geotag'][0]
listing['longitude'] = result['geotag'][1]
# decide if we want to notify about this listing
# https://stackoverflow.com/questions/2783969/compare-string-with-all-values-in-array
if any(x in listing['neighborhood']
for x in settings.neighborhood_blacklist):
notify = False
else:
notify = True
# check if the listing is a duplicate
if database.get_record(listing['craigslist_id']):
log.info('Found duplicate record with ID {}, skipping'.format(
listing['craigslist_id']))
continue # if duplicate we assume we've procsessed this listing so just skip it
# otherwise we should save the listing and notify if applicable
else:
log.info('{} looks like a new listing, processing'.format(
'craigslist_id'))
# get the map image from Mapbox
# we do this here instead of above to limit the number of API requests made to Mapbox
listing['map_image'] = get_map(listing['latitude'],
listing['longitude'])
database.insert_record(listing)
if notify is True:
send_notification(listing)
database.mark_as_notified(listing['craigslist_id'])
def _post(self):
request = _get_json(self.request.body)
uid = request['uid']
_logger.info("Proceeding with retrieval request for '%(uid)s'..." % {
'uid': uid,
})
record = database.get_record(uid)
if not record:
self.send_error(404)
return
trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
if not trust.read:
self.send_error(403)
return
current_time = int(time.time())
record['physical']['atime'] = current_time
record['stats']['accesses'] += 1
for policy in ('delete', 'compress'):
if 'stale' in record['policy'][policy]:
record['policy'][policy]['staleTime'] = current_time + record['policy'][policy]['stale']
database.update_record(record)
fs = state.get_filesystem(record['physical']['family'])
try:
data = fs.get(record)
except filesystem.FileNotFoundError as e:
_logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
'uid': uid,
})
self.send_error(404)
return
else:
_logger.debug("Evaluating decompression requirements...")
applied_compression = record['physical']['format'].get('comp')
supported_compressions = (c.strip() for c in (self.request.headers.get('Media-Storage-Supported-Compression') or '').split(';'))
if applied_compression and not applied_compression in supported_compressions: #Must be decompressed first
data = compression.get_decompressor(applied_compression)(data)
applied_compression = None
_logger.debug("Returning entity...")
self.set_header('Content-Type', record['physical']['format']['mime'])
if applied_compression:
self.set_header('Media-Storage-Applied-Compression', applied_compression)
while True:
chunk = data.read(_CHUNK_SIZE)
if chunk:
self.write(chunk)
else:
break
def _post(self):
request = _get_json(self.request.body)
uid = request['uid']
_logger.info("Proceeding with description request for '%(uid)s'..." % {
'uid': uid,
})
record = database.get_record(uid)
if not record:
self.send_error(404)
return
trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
if not trust.read:
self.send_error(403)
return
_logger.debug("Describing entity...")
record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record)
record['uid'] = record['_id']
del record['_id']
del record['physical']['minRes']
del record['keys']
return record
def gets_url(original_id):
return database.get_record(original_id)