def uploadToS3(fobj, filename): log.debug('Filename is: {0}'.format(filename)) conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) bucket = conn.get_bucket('0nk38gf20cct6vytbg02_tfl_data') try: key = Key(bucket) key.key = 'xml/'+filename log.debug('Sending file {0} with key {1}'.format(filename, key.key)) key.set_contents_from_file(fobj, reduced_redundancy=True) sendToSqs([filename], 'tfl_queue_parse') except: log.exception('Unable to upload file {0}'.format(filename))
def midnightRollHDF(): olddir = os.path.join(HDF_DIR, 'store.h5') newdir = os.path.join(HDF_DIR, datetime.datetime.now().strftime('%Y%m%d-%H%M%S.h5')) #set indicies log.info('Compressing hdf file {0} to new hdf file {1}'.format(olddir, newdir)) process = subprocess.Popen(("ptrepack", "--chunkshape=auto", "--propindexes", "--complevel=9", "--complib=bzip2", olddir, newdir), stdout=subprocess.PIPE) log.info('compression is complete with message {0}'.format(process.communicate()[0])) sendToSqs([newdir], 'tfl_queue_upload') log.info('New compressed hdf file sent to upload, deleting old hdf') try: if os.path.isfile(olddir): os.unlink(olddir) except: log.exceptio('Delete of old hdf failed!')
def sendToSqs(self, filenames, name): return sendToSqs(filenames, name)