def __init__(self):
self.uuid = str(uuid4())
self.content_type = 'application/octet-stream'
self.epoch = epoch()
self.added = epoch()
self.filename = None
self.ref = None
self.description = None
self.author = None
self.links = {}
self.metadata = {}
def __init__(self):
self.uuid = str(uuid4())
self.content_type = 'application/octet-stream'
self.epoch = epoch()
self.added = epoch()
self.filename = None
self.ref = None
self.description = None
self.author = None
self.links = {}
self.metadata = {}
def main(update=False, dry=False, pages=1):
for tweet in iter_tweets(pages=pages):
existing = lookup_record(tweet)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.epoch = epoch(parse(tweet[u'created_at']))
r.content_type = 'application/json'
r.ref = 'https://twitter.com/kennethreitz/status/{}'.format(
tweet['id'])
r.author = 'Kenneth Reitz'
r.description = u'Tweet: {}'.format(tweet['text'])
r.metadata['service'] = 'twitter'
r.metadata['id'] = tweet['id']
r.metadata['text'] = tweet['text']
r.metadata['retweeted'] = tweet['retweeted']
r.metadata['retweet_count'] = tweet['retweet_count']
r.metadata['entities'] = tweet['entities']
r.metadata['coordinates'] = tweet.get('coordinates')
if not dry:
r.save(archive=True)
r.upload_task.delay(r, data=json.dumps(tweet), archive=True)
print r
def main(update=False, dry=False):
for photo in iter_photos():
existing = lookup_record(photo)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.content_type = 'image/jpeg'
r.ref = 'http://www.flickr.com/photos/{}/{}'.format(
photo['pathalias'], photo['id'])
r.description = u'Flickr photo. {}'.format(photo['title'])
r.author = 'Kenneth Reitz'
r.links['src'] = photo['url_o']
r.epoch = int(photo[u'dateupload']) * 1000
r.metadata['id'] = photo['id']
r.metadata['service'] = 'flickr'
r.metadata['title'] = photo['title']
r.metadata['taken'] = epoch(parse(photo[u'datetaken']))
r.metadata['longitude'] = photo['longitude']
r.metadata['latitude'] = photo['latitude']
print r
if not dry:
r.save(archive=True)
r.upload_task.delay(r, url=photo['url_o'], archive=True)
def main(update=False, dry=False, pages=1):
for tweet in iter_tweets(pages=pages):
existing = lookup_record(tweet)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.epoch = epoch(parse(tweet[u'created_at']))
r.content_type = 'application/json'
r.ref = 'https://twitter.com/kennethreitz/status/{}'.format(tweet['id'])
r.author = 'Kenneth Reitz'
r.description = u'Tweet: {}'.format(tweet['text'])
r.metadata['service'] = 'twitter'
r.metadata['id'] = tweet['id']
r.metadata['text'] = tweet['text']
r.metadata['retweeted'] = tweet['retweeted']
r.metadata['retweet_count'] = tweet['retweet_count']
r.metadata['entities'] = tweet['entities']
r.metadata['coordinates'] = tweet.get('coordinates')
if not dry:
r.save(archive=True)
r.upload_task.delay(r, data=json.dumps(tweet), archive=True)
print r
def main(update=False, dry=False):
for photo in iter_photos():
existing = lookup_record(photo)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.content_type = 'image/jpeg'
r.ref = 'http://500px.com/photo/{}'.format(photo['id'])
r.description = u'500px: {}, '.format(photo['name'], photo['description'])
r.author = 'Kenneth Reitz'
r.epoch = epoch(parse(photo[u'created_at']))
r.metadata['service'] = '500px'
r.metadata['height'] = photo['height']
r.metadata['width'] = photo['width']
r.metadata['id'] = photo['id']
r.metadata['name'] = photo['name']
r.metadata['description'] = photo['width']
r.metadata['nsfw'] = photo['nsfw']
r.metadata['src'] = photo['image_url'].replace('2.jpg', '4.jpg')
if not dry:
r.save(archive=True)
r.upload_task.delay(r, url=photo['image_url'].replace('2.jpg', '4.jpg'), archive=True)
print r
def main(update=False, dry=False):
for photo in iter_photos():
existing = lookup_record(photo)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.content_type = 'image/jpeg'
r.ref = 'http://www.flickr.com/photos/{}/{}'.format(photo['pathalias'], photo['id'])
r.description = u'Flickr photo. {}'.format(photo['title'])
r.author = 'Kenneth Reitz'
r.links['src'] = photo['url_o']
r.epoch = int(photo[u'dateupload'])*1000
r.metadata['id'] = photo['id']
r.metadata['service'] = 'flickr'
r.metadata['title'] = photo['title']
r.metadata['taken'] = epoch(parse(photo[u'datetaken']))
r.metadata['longitude'] = photo['longitude']
r.metadata['latitude'] = photo['latitude']
print r
if not dry:
r.save(archive=True)
r.upload_task.delay(r, url=photo['url_o'], archive=True)
def main(update=False, dry=False):
for photo in iter_photos():
existing = lookup_record(photo)
if existing:
print 'Existing:',
if not update:
print '{0}. \nExiting.'.format(existing)
return
r = existing or blackbox.Record()
r.content_type = 'image/jpeg'
r.ref = 'http://500px.com/photo/{}'.format(photo['id'])
r.description = u'500px: {}, '.format(photo['name'],
photo['description'])
r.author = 'Kenneth Reitz'
r.epoch = epoch(parse(photo[u'created_at']))
r.metadata['service'] = '500px'
r.metadata['height'] = photo['height']
r.metadata['width'] = photo['width']
r.metadata['id'] = photo['id']
r.metadata['name'] = photo['name']
r.metadata['description'] = photo['width']
r.metadata['nsfw'] = photo['nsfw']
r.metadata['src'] = photo['image_url'].replace('2.jpg', '4.jpg')
if not dry:
r.save(archive=True)
r.upload_task.delay(r,
url=photo['image_url'].replace(
'2.jpg', '4.jpg'),
archive=True)
print r
