def add_key (alg, key, nm):
try:
name, machine = nm.split('@')
except ValueError:
return False
name = name
d = './' + keydir + '/' + machine
kf = d + '/' + prepend + name + '.pub'
if not mre.match (machine) or not nre.match (name) or not are.match (alg):
return False
if os.path.exists(kf):
print 'ignoring duplicate key for:', kf
return True # we do this so that we don't leak info
f = NamedTemporaryFile(delete=False)
f.file.write ('%s %s %s@%s\n' % (alg, key, name, machine))
f.close()
p = Popen(['ssh-vulnkey', f.name], stdin=PIPE, stdout=PIPE, stderr=PIPE)
p.stdin.close()
if p.stderr.read().__len__() > 1:
f.unlink (f.name)
return False
if not os.path.exists (d):
os.makedirs (d)
shutil.move (f.name, kf)
print "Imported", kf
return True
def test_dump(self):
cbd1 = CitiBikeData(source_url=self.test_data_url)
self.assert_data_loaded(cbd1)
js = StringIO()
cbd1.dump(js)
self.assert_data_loaded(cbd1)
self.assertGreater(len(js.getvalue()), 0)
js.reset()
cbd2 = CitiBikeData(load_on_init=False)
self.assert_data_not_loaded(cbd2)
cbd2.load(js)
self.assert_data_loaded(cbd2)
self.assertDictEqual(cbd1.json, cbd2.json)
ntf = NamedTemporaryFile(delete=False)
cbd1.dump(ntf)
self.assert_data_loaded(cbd1)
self.assertGreater(len(js.getvalue()), 0)
ntf.close()
cbd3 = CitiBikeData(source_url="file:"+ntf.name)
self.assert_data_loaded(cbd3)
self.assertDictEqual(cbd1.json, cbd3.json)
ntf.unlink(ntf.name) # delete file
def atomic_write(content, target):
t = NamedTemporaryFile(dir="/tmp", delete=False)
t.file.write(content)
t.file.flush()
t.close()
copy(t.name, target)
t.unlink(t.name)
def export(self, out_f=None, format='mp3'):
out_f = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
data = NamedTemporaryFile(mode="wb", delete=False)
wave_data = wave.open(data)
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
wave_data.setnframes(self.frame_count())
wave_data.writeframesraw(self._data)
wave_data.close()
output = NamedTemporaryFile(mode="w+")
# read stdin / write stdout
subprocess.call(['ffmpeg',
'-y', # always overwrite existing files
"-f", "wav", "-i", data.name, # input options (filename last)
"-f", format, output.name, # output options (filename last)
],
# make ffmpeg shut up
stderr=open(os.devnull))
output.seek(0)
out_f.write(output.read())
data.unlink(data.name)
out_f.seek(0)
return out_f
def _save_samba_share(conf):
temp = NamedTemporaryFile('w', delete=False)
conf.write(temp)
temp.close()
bkp_date = datetime.datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
process = run_as_root('cp "/etc/samba/smb.conf" '\
'"/etc/samba/smb.conf-{0}.bkp"'
.format(bkp_date))
process.expect(pexpect.EOF)
process = run_as_root('cp "{0}" "/etc/samba/smb.conf"'
.format(temp.name))
process.expect(pexpect.EOF)
process = run_as_root('chmod 644 /etc/samba/smb.conf')
process.expect(pexpect.EOF)
process = run_as_root('chown root:root /etc/samba/smb.conf')
process.expect(pexpect.EOF)
temp.unlink(temp.name)
def launcher_hadoop_job(self,
data_type,
input,
output,
result_companyId,
map_tasks=8,
red_tasks=8):
"""Runs the Hadoop job uploading task configuration"""
# create report to save on completion or error
report = {
'started_at': datetime.now(),
'state': 'launched',
'input': input
}
# Create temporary file to upload with json extension to identify it in HDFS
job_extra_config = self.config.copy()
job_extra_config.update({'companyId': result_companyId})
f = NamedTemporaryFile(delete=False, suffix='.json')
f.write(json.dumps(job_extra_config))
f.close()
self.logger.debug(
'Created temporary config file to upload into hadoop and read from job: %s'
% f.name)
# create hadoop job instance adding file location to be uploaded
if data_type == "billing":
mr_job = MRJob_clean_billing_data(args=[
'-r', 'hadoop', 'hdfs://' + input, '--file', f.name, '-c',
'module_edinet/edinet_clean_daily_data_etl/mrjob.conf',
'--output-dir', 'hdfs://' + output, '--jobconf',
'mapred.job.name=edinet_clean_daily_data_etl_billing',
'--jobconf', 'mapred.reduce.tasks={}'.format(self.num_reducers)
])
elif data_type == "metering":
mr_job = MRJob_clean_metering_data(args=[
'-r', 'hadoop', 'hdfs://' + input, '--file', f.name, '-c',
'module_edinet/edinet_clean_daily_data_etl/mrjob.conf',
'--output-dir', 'hdfs://' + output, '--jobconf',
'mapred.job.name=edinet_clean_daily_data_etl_metering',
'--jobconf', 'mapred.reduce.tasks={}'.format(self.num_reducers)
])
else:
raise Exception(
"The job with data type {} can not be treated".format(
data_type))
with mr_job.make_runner() as runner:
try:
runner.run()
except Exception as e:
f.unlink(f.name)
raise Exception(
'Error running MRJob process using hadoop: {}'.format(e))
f.unlink(f.name)
self.logger.debug(
'Temporary config file uploaded has been deleted from FileSystem')
report['finished_at'] = datetime.now()
report['state'] = 'finished'
return report
def test_pipehandler(self):
tmp = NamedTemporaryFile(delete=False)
tmp.close()
Handler = HandlerFactory("pipe", {"path":tmp.name})
h = Handler("test")
h([])
with open(tmp.name,"r") as f:
self.assertEqual(f.read(), "test")
tmp.unlink(tmp.name)
def launcher_hadoop_job(self,
type,
input,
company=None,
devices=None,
stations=None,
map_tasks=8,
red_tasks=8):
"""Runs the Hadoop job uploading task configuration"""
# create report to save on completion or error
report = {
'started_at': datetime.now(),
'state': 'launched',
'input': input
}
# Create temporary file to upload with json extension to identify it in HDFS
job_extra_config = self.config.copy()
job_extra_config.update({
'devices': devices,
'company': company,
'stations': stations,
'task_id': self.task_UUID
})
f = NamedTemporaryFile(delete=False, suffix='.json')
f.write(json.dumps(job_extra_config))
f.close()
report['config_temp_file'] = f.name
self.logger.debug(
'Created temporary config file to upload into hadoop and read from job: %s'
% f.name)
# create hadoop job instance adding file location to be uploaded
mr_job = MRJob_align(args=[
'-r', 'hadoop', 'hdfs://' + input, '--file', f.name, '-c',
'module_edinet/edinet_baseline_hourly_module/mrjob.conf',
'--jobconf', 'mapred.job.name=edinet_baseline_hourly_module',
'--jobconf', 'mapred.reduce.tasks={}'.format(self.num_reducers)
])
# mr_job = MRJob_align(args=['-r', 'hadoop', 'hdfs://'+input, '--file', f.name, '--output-dir', '/tmp/prova_dani', '--python-archive', path.dirname(lib.__file__)]) # debugger
with mr_job.make_runner() as runner:
try:
runner.run()
except Exception as e:
f.unlink(f.name)
raise Exception(
'Error running MRJob process using hadoop: {}'.format(e))
f.unlink(f.name)
self.logger.debug(
'Temporary config file uploaded has been deleted from FileSystem')
report['finished_at'] = datetime.now()
report['state'] = 'finished'
return report
def test_clbhandler(self):
tmp = NamedTemporaryFile(delete=False)
tmp.file.write("def raise_(): raise FutureWarning\n")
tmp.file.write("def raise__(smt): raise smt")
tmp.close()
Handler = HandlerFactory("callback", {"path": tmp.name})
self.assertRaises(FutureWarning, Handler("raise_()"), [])
self.assertRaises(FutureWarning, Handler("raise__({0})"), [FutureWarning])
tmp.unlink(tmp.name)
def export(self, out_f=None, format='mp3', codec=None):
out_f = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
# for wav output we can just write the data directly to out_f
if format == "wav":
data = out_f
else:
data = NamedTemporaryFile(mode="wb", delete=False)
wave_data = wave.open(data, 'wb')
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
wave_data.setnframes(self.frame_count())
wave_data.writeframesraw(self._data)
wave_data.close()
# for wav files, we're done (wav data is written directly to out_f)
if format == 'wav':
return out_f
output = NamedTemporaryFile(mode="w+")
# build call args
args = [
self.ffmpeg,
'-y', # always overwrite existing files
"-f",
"wav",
"-i",
data.name, # input options (filename last)
]
if codec is not None:
# force audio encoder
args.extend(["-acodec", codec])
args.extend([
"-f",
format,
output.name, # output options (filename last)
])
# read stdin / write stdout
subprocess.call(
args,
# make ffmpeg shut up
stderr=open(os.devnull))
output.seek(0)
out_f.write(output.read())
data.unlink(data.name)
out_f.seek(0)
return out_f
def upload(self, src_name, dst_name, compress=True):
if not compress:
self._upload(src_name, dst_name)
fout = NamedTemporaryFile(suffix='.gz', mode='wb', delete=False)
try:
fout.close()
logging.debug('Compressing file %s...', src_name)
with \
open(src_name, 'rb') as fin, \
closing(gzip.GzipFile(fout.name, mode='wb')) as gzout:
for chunk in iterchunks(fin):
gzout.write(chunk)
return self._upload(fout.name, dst_name + '.gz')
finally:
fout.unlink(fout.name)
def upload(self, src_name, dst_name, compress=True):
if not compress:
self._upload(src_name, dst_name)
fout = NamedTemporaryFile(suffix='.gz', mode='wb', delete=False)
try:
fout.close()
logging.debug('Compressing file %s...', src_name)
with \
open(src_name, 'rb') as fin, \
closing(gzip.GzipFile(fout.name, mode='wb')) as gzout:
for chunk in iterchunks(fin):
gzout.write(chunk)
return self._upload(fout.name, dst_name + '.gz')
finally:
fout.unlink(fout.name)
def export(self, out_f=None, format='mp3', codec=None):
out_f = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
# for wav output we can just write the data directly to out_f
if format == "wav":
data = out_f
else:
data = NamedTemporaryFile(mode="wb", delete=False)
wave_data = wave.open(data, 'wb')
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
wave_data.setnframes(self.frame_count())
wave_data.writeframesraw(self._data)
wave_data.close()
# for wav files, we're done (wav data is written directly to out_f)
if format == 'wav':
return out_f
output = NamedTemporaryFile(mode="w+")
# build call args
args =[self.ffmpeg,
'-y', # always overwrite existing files
"-f", "wav", "-i", data.name, # input options (filename last)
]
if codec is not None:
# force audio encoder
args.extend(["-acodec", codec])
args.extend([
"-f", format, output.name, # output options (filename last)
])
# read stdin / write stdout
subprocess.call(args,
# make ffmpeg shut up
stderr=open(os.devnull)
)
output.seek(0)
out_f.write(output.read())
data.unlink(data.name)
out_f.seek(0)
return out_f
def hadoop_job(self, input, companyId):
"""Runs the Hadoop job uploading task configuration"""
# create report to save on completion or error
report = {
'started_at': datetime.now(),
'state': 'launched',
'input': input
}
# Create temporary file to upload with json extension to identify it in HDFS
f = NamedTemporaryFile(delete=False, suffix='.json')
f.write(json.dumps(self.config))
f.close()
self.logger.debug(
'Created temporary config file to upload into hadoop and read from job: {}'
.format(f.name))
# create hadoop job instance adding file location to be uploaded
dtnow = datetime.now()
str_dtnow = dtnow.strftime("%Y%m%d%H%M")
mr_job = Hadoop_ETL(args=[
'-r', 'hadoop', input, '--file', f.name, '--output-dir', "{}/{}/{}"
.format(self.config['error_measures'], str(companyId), str_dtnow),
'-c', 'module_edinet/edinet_billing_measures_etl/mrjob.conf',
'--jobconf', 'mapred.job.name=edinet_billing_measures_etl'
])
# mr_job = Hadoop_ETL(args=['-r', 'hadoop', input, '--file', f.name, '--python-archive', path.dirname(lib.__file__)])
with mr_job.make_runner() as runner:
try:
runner.run()
except Exception as e:
f.unlink(f.name)
raise Exception(
'Error running MRJob ETL process using hadoop: {}'.format(
e))
f.unlink(f.name)
self.logger.debug(
'Temporary config file uploaded has been deleted from FileSystem')
report['finished_at'] = datetime.now()
report['state'] = 'finished'
return report
def validate_sparse_support(directory):
SIZE = 1024 * 1024 # 1 MiB temp file seems reasonable
if not directory.exists():
raise FileNotFoundError(f"No such file or directory: '{directory}'")
elif not directory.is_dir():
raise ValueError(f"Expected '{directory}' to be a directory")
# Try to write a small sparse file in the same directory as `filepath` and then read
# the actual size to check that it's sparse
temp_file = NamedTemporaryFile(dir=directory, delete=False)
temp_file.close()
temp_file = Path(temp_file.name)
write_sparse_file(temp_file, b"~Testing~", SIZE)
temp_file_size = size_on_disk(temp_file)
if temp_file_size is None:
return False
is_sparse = temp_file_size < SIZE
temp_file.unlink()
return is_sparse
def upload(self, src_name, dst_name, compress=True, use_gzip=False):
if compress:
fout = NamedTemporaryFile(suffix='.gz', mode='wb', delete=False)
try:
if use_gzip:
logging.debug('Compressing file %s with gzip...', src_name)
p = subprocess.Popen(["gzip", '-c', src_name], stdout=fout)
assert p.wait() == 0, 'Gzip compression failed'
fout.close()
return self._upload(fout.name, dst_name + '.gz')
else:
fout.close()
logging.debug('Compressing file %s...', src_name)
with \
open(src_name, 'rb') as fin, \
closing(gzip.GzipFile(fout.name, mode='wb')) as gzout:
for chunk in iterchunks(fin):
gzout.write(chunk)
return self._upload(fout.name, dst_name + '.gz')
finally:
fout.unlink(fout.name)
else:
self._upload(src_name, dst_name)
def _save_samba_share(conf):
temp = NamedTemporaryFile('w', delete=False)
conf.write(temp)
temp.close()
bkp_date = datetime.datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
process = run_as_root('cp "/etc/samba/smb.conf" '\
'"/etc/samba/smb.conf-{0}.bkp"'
.format(bkp_date))
process.expect(pexpect.EOF)
process = run_as_root('cp "{0}" "/etc/samba/smb.conf"'.format(temp.name))
process.expect(pexpect.EOF)
process = run_as_root('chmod 644 /etc/samba/smb.conf')
process.expect(pexpect.EOF)
process = run_as_root('chown root:root /etc/samba/smb.conf')
process.expect(pexpect.EOF)
temp.unlink(temp.name)
def handle(self, *args, **options):
if not GOOGLE_DOCS_ACCOUNT and GOOGLE_DOCS_PASSWORD and GOOGLE_DOCS_RESOURCE_ID:
raise CommandError('You must set both GOOGLE_DOCS_ACCOUNT, GOOGLE_DOCS_PASSWORD and GOOGLE_DOCS_RESOURCE_ID in your settings file.')
verbosity = int(options.get('verbosity', 1))
output_all = options.get('output_all')
dry_run = options.get('dry_run')
fields = ('email', 'first_name', 'last_name', 'phone', 'city', 'state', 'zipcode', 'is_a', 'broadcasters', 'date_created', 'share_info')
profile_list = NonUserProfile.objects.order_by('-date_created')
if not output_all:
profile_list = profile_list.filter(share_info=True)
if len(profile_list):
if verbosity > 1:
self.stdout.write('{0} signups to record.'.format(len(profile_list)))
fp = NamedTemporaryFile(delete=False)
writer = csv.DictWriter(fp, fields)
writer.writeheader()
for signup in profile_list:
output = {
'email': signup.email,
'first_name': signup.first_name,
'last_name': signup.last_name,
'phone': signup.phone,
'city': signup.city,
'state': signup.state,
'zipcode': signup.zipcode,
'is_a': signup.is_a,
'date_created': signup.date_created.strftime('%m/%d/%Y %H:%M:%S'),
'share_info': signup.share_info
}
extra_fields_data = signup.extra_fields
for extra_field in SIGNUP_EXTRA_FIELDS:
input_val = None
if isinstance(extra_fields_data[extra_field], list):
input_val = ', '.join(extra_fields_data[extra_field])
else:
input_val = extra_fields_data[extra_field]
output[extra_field] = input_val
writer.writerow(output)
if dry_run:
self.stdout.write('Row created:\n{0}\n'.format('|'.join([str(output[f]) for f in fields])))
del(writer)
if not dry_run:
client = gdata.docs.client.DocsClient()
login_token = client.ClientLogin(GOOGLE_DOCS_ACCOUNT, GOOGLE_DOCS_PASSWORD, 'politicaladsleuth')
fp.close()
media = gdata.data.MediaSource(file_path=fp.name, content_type='text/csv')
try:
resource = client.get_resource_by_id(GOOGLE_DOCS_RESOURCE_ID)
updated_resource = client.update_resource(resource, media=media, update_metadata=False, new_revision=True)
self.stdout.write('Data uploaded to "%s"\n'.format(updated_resource.title.text))
except gdata.client.RequestError as e:
self.stderr.write(e.message + '\n')
self.stdout.write('****Upload may have succeeded despite an InvalidEntryException error****\n')
fp.close()
fp.unlink(fp.name)
else:
self.stdout.write('No signups for the given parameters\n')
class FacadeTest(TestCase):
def setUp(self):
cfg = ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'foo', 'bar')
cfg.set('some_section', 'host', 'foo')
cfg.set('some_section', 'port', '29192')
self.config_file = NamedTemporaryFile(mode='w', delete=False)
cfg.write(self.config_file)
self.config_file.close()
self.env_file = NamedTemporaryFile(mode='w', delete=False)
cfg = ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'host', 'SOME_HOST_NAME')
cfg.set('some_section', 'port', 'SOME_PORT_NAME')
cfg.add_section('other_section')
cfg.set('other_section', 'foo', 'bar')
cfg.write(self.env_file)
self.env_file.close()
def tearDown(self):
self.config_file.unlink(self.config_file.name)
self.env_file.unlink(self.env_file.name)
def test_not_existing_config(self):
configuration = get_configuration('non_existing_section')
self.assertIsNone(configuration)
def test_not_existing_variables(self):
configuration = get_configuration('messaging',
config_file='non-existing')
self.assertIsNone(configuration)
def test_existing_config(self):
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'foo')
self.assertEquals(configuration['port'], '29192')
self.assertFalse('foo' in configuration)
def test_existing_env(self):
os.environ['SOME_HOST_NAME'] = 'bar'
os.environ['SOME_PORT_NAME'] = '6661'
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'bar')
self.assertEquals(configuration['port'], '6661')
self.assertFalse('foo' in configuration)
os.environ.pop('SOME_HOST_NAME')
os.environ.pop('SOME_PORT_NAME')
def test_not_in_config(self):
configuration = get_configuration(section_name='other_section',
config_file=self.config_file.name,
variables=self.env_file.name)
self.assertIsNone(configuration)
def store(self, file=None, content=None, ctype=None, **kwd):
"""save a file-like item"""
if content is None and not hasattr(file, 'read'):
raise TypeError('invalid file-like object')
data = content if content is not None else file.read()
size = len(data)
ext = guessImageType(data[:32])
if ext is None:
raise ValueError('invalid image file')
hashes = [md5(data).hexdigest()]
_exists_id = self.exists(hashed=hashes[0])
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print('id {} or hash {} exists!!'.format(id, hashes[0]))
#raise DuplicateError('already exists')
return [True, id, filename]
ids = [_make_id(hashes[0])]
if 'id' in kwd and kwd['id'] and kwd['id'] not in ids:
ids += [kwd['id']]
from image import SimpImage, MIN_QUALITY
max_file_size = int(self.get_config('max_file_size'))
max_jpeg_quality = int(self.get_config('max_jpeg_quality'))
max_width = int(self.get_config('max_width'))
max_height = int(self.get_config('max_height'))
if size > max_file_size: max_jpeg_quality -= 1
if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY
im = SimpImage(blob=data)
meta = im.meta
if meta['width'] > max_width or meta['height'] > max_height:
if self.get_config('auto_scale') and im.thumbnail(
max_width, max_height):
if im.format == 'JPEG' and im.quality > max_jpeg_quality:
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
print im.meta
print 'new scaled size {}'.format(size)
hashes += [md5(data).hexdigest()]
else:
raise ValueError(
'file: {} dimension {}x{} is too big, max is {}x{}'.format(
kwd['name'] if 'name' in kwd else '', meta['width'],
meta['height'], max_width, max_height))
if im.format == 'JPEG':
if im.quality > max_jpeg_quality:
print 'quality {} is too high, hash {}'.format(
im.quality, hashes[0])
from tempfile import NamedTemporaryFile
_tmp = NamedTemporaryFile('w+b',
dir=self.get_config('temp_root'),
delete=False)
_tmp.file.close()
save_file(_tmp.name, blob=data)
if jpegoptim(_tmp.name):
fp = open(_tmp.name)
data = fp.read()
size = len(data)
# print 'new optimized size {}'.format(size)
fp.close()
_tmp.unlink(_tmp.name)
del im
im = SimpImage(blob=data)
meta = im.meta
hashes += [md5(data).hexdigest()]
else:
raise EnvironmentError(
'jpeg qualty is too high, or need jpegoptim')
elif im.format == 'PNG' and self.get_config('force_jpeg'):
im.format = 'JPEG'
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
hashes += [md5(data).hexdigest()]
ext = 'jpg'
meta = im.meta
del im
if (size > max_file_size):
raise ValueError('file: {} size {} is too big, max is {}'.format(
kwd['name'] if 'name' in kwd else '', size, max_file_size))
hashed = hashes[len(hashes) - 1] #md5(data).hexdigest()
# print ('md5 hash: {}'.format(hashed))
# TODO: add for support (md5 + size) id
id = _make_id(hashed)
# print ('new filename: %r' % filename)
# TODO: fix for support s3 front browse
_exists_id = self.exists(id) or self.exists(hashed=hashed)
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print('id {} or hash {} exists!!'.format(id, hashed))
#raise DuplicateError('already exists')
return [True, id, filename]
filename = _make_filename(id, ext)
# print ('id: {}'.format(id))
# if ctype is None or ctype == '':
from _util import guess_mimetype
ctype = guess_mimetype(filename)
# save to mongodb
spec = {
'_id': id,
'filename': filename,
'hash': hashes,
'mime': ctype,
'size': size,
'meta': meta,
'ids': ids
}
if 'name' in kwd and isinstance(kwd['name'], (str, unicode)):
spec['name'] = kwd['name']
for k in ['created', 'app_id']:
if k in kwd and kwd[k]:
spec[k] = kwd[k]
if self._store_exists(id, filename=filename):
self._save_meta(id, spec)
return [True, id, filename]
rr = self._put(data, **spec)
if rr:
return [True, rr, filename]
class ConfigManagerTest(unittest.TestCase):
def setUp(self):
cfg = ConfigParser.ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'host', 'foo')
cfg.set('some_section', 'port', '29192')
cfg.set('some_section', 'backend', 'tests.test_config_manager.Backend')
self.config_file = NamedTemporaryFile(mode='w', delete=False)
cfg.write(self.config_file)
self.config_file.close()
self.env_file = NamedTemporaryFile(mode='w', delete=False)
cfg = ConfigParser.ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'host', 'SOME_HOST_NAME')
cfg.set('some_section', 'port', 'SOME_PORT_NAME')
cfg.add_section('other_section')
cfg.set('other_section', 'foo', 'bar')
cfg.write(self.env_file)
self.env_file.close()
def tearDown(self):
self.config_file.unlink(self.config_file.name)
self.env_file.unlink(self.env_file.name)
def test_not_existing_section(self):
configuration = get_configuration('not-existing-section',
config_file=self.config_file.name)
self.assertIsNone(configuration)
def test_existing_config(self):
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'foo')
self.assertEquals(configuration['port'], '29192')
self.assertFalse('foo' in configuration)
def test_existing_env(self):
os.environ['SOME_HOST_NAME'] = 'bar'
os.environ['SOME_PORT_NAME'] = '6661'
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'bar')
self.assertEquals(configuration['port'], '6661')
self.assertFalse('foo' in configuration)
os.environ.pop('SOME_HOST_NAME')
os.environ.pop('SOME_PORT_NAME')
def test_get_backend_class(self):
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
backend_class = get_backend_class(configuration)
self.assertTrue('Backend' in str(backend_class))
def module_task(self, params):
self.logger.info('Starting Module for edinet baseline...')
"""CHECK INCONSISTENCIES IN params"""
try:
result_companyId = params['result_companyId']
ts_to = params['ts_to']
ts_from = params[
'ts_from'] if 'ts_from' in params else date_n_month(
ts_to, -24)
energyTypeList = params['type'] if 'type' in params else []
except KeyError as e:
raise Exception(
'Not enough parameters provided to module: {}'.format(e))
######################################################################################################################################################################################
""" GET DATA FROM MONGO TO MAKE QUERYS """
######################################################################################################################################################################################
if not energyTypeList:
energyTypeList = list(
set([
x['type']
for x in self.mongo['readings'].find({}, {'type': 1})
]))
#####################################################################################################################################################################################
""" LOAD DATA FROM HIVE """
######################################################################################################################################################################################
self.logger.info('Extracting data from mongodb')
# setting variables for readability
collection = self.config['mongodb']['modelling_units_collection']
self.logger.debug('Querying for modelling units in MongoDB')
cursor = self.mongo[collection].find({})
device_key = {}
stations = {}
for item in cursor:
if len(item['devices']) > 0: # to avoid empty list of devices
for dev in item['devices']:
stations[str(dev['deviceId'].encode('utf-8'))] = str(
item['stationId']) if 'stationId' in item else None
if str(dev['deviceId'].encode(
'utf-8')) in device_key.keys():
device_key[str(
dev['deviceId'].encode('utf-8'))].append(
str(item['modellingUnitId']) + '~' +
str(item['devices']))
else:
device_key[str(dev['deviceId'].encode('utf-8'))] = [
str(item['modellingUnitId']) + '~' +
str(item['devices'])
]
self.logger.info('A mongo query process has loaded {} devices'.format(
len(device_key.keys())))
######################################################################################################################################################################################
""" HIVE QUERY TO PREPARE DATA FOR MRJOB """
######################################################################################################################################################################################
# create a table to link devices with stations
device_stations_df = pd.DataFrame(data={
"deviceId": stations.keys(),
"stationId": stations.values()
},
columns=["deviceId", "stationId"])
f = NamedTemporaryFile(delete=False, suffix='.csv')
device_stations_df.to_csv(f.name, header=None, index=None)
f.close()
call([
"hadoop", "fs", "-mkdir", "-p", f.name,
self.config['paths']['stations']
])
call([
"hadoop", "fs", "-copyFromLocal", f.name,
self.config['paths']['stations']
])
f.unlink(f.name)
device_stations = create_hive_module_input_table(
self.hive,
'edinet_device_stations_table',
self.config['paths']['stations'], [('deviceId', 'string'),
('stationId', 'string')],
self.task_UUID,
sep=",")
self.context.add_clean_hive_tables(device_stations)
# create a table with the devices values
fields = [('deviceId', 'string'), ('ts', 'int'), ('value', 'float'),
('energyType', 'string'), ('source', 'string'),
('temperature', 'string')]
location = self.config['paths']['measures']
input_table = create_hive_module_input_table(self.hive,
'edinet_baseline_input',
location, fields,
self.task_UUID)
#add input table to be deleted after execution
self.context.add_clean_hive_tables(input_table)
qbr = RawQueryBuilder(self.hive)
sentence = """
INSERT OVERWRITE TABLE {input_table}
SELECT a.deviceId, a.ts, a.value, a.energyType, a.source, c.temperature FROM
(SELECT ai.deviceid as deviceId, ai.ts as ts, ai.value as value, ai.energyType as energyType, ai.source as source FROM edinet_hourly_consumption ai
WHERE
ai.ts >= UNIX_TIMESTAMP("{ts_from}","yyyy-MM-dd HH:mm:ss") AND
ai.ts <= UNIX_TIMESTAMP("{ts_to}","yyyy-MM-dd HH:mm:ss")) a
JOIN {device_stations} b on a.deviceId==b.deviceId
JOIN edinet_meteo c on b.stationId==c.stationId and SUBSTR(FROM_UNIXTIME(a.ts), 1, 13) == SUBSTR(FROM_UNIXTIME(c.ts), 1, 13)
""".format(input_table=input_table,
ts_from=ts_from,
ts_to=ts_to,
device_stations=device_stations)
self.logger.debug(sentence)
qbr.execute_query(sentence)
######################################################################################################################################################################################
""" SETUP MAP REDUCE JOB """
######################################################################################################################################################################################
self.logger.info('Getting')
try:
# Launch MapReduce job
## Buffered measures to HBase
self.logger.debug('MRJob Align')
self.launcher_hadoop_job('align', location, result_companyId,
device_key, stations)
except Exception as e:
raise Exception('MRJob ALIGN process job has failed: {}'.format(e))
self.logger.info('Module EDINET_baseline execution finished...')
class ConfigManagerTest(unittest.TestCase):
def setUp(self):
cfg = ConfigParser.ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'host', 'foo')
cfg.set('some_section', 'port', '29192')
cfg.set('some_section', 'backend', 'tests.test_config_manager.Backend')
self.config_file = NamedTemporaryFile(mode='w', delete=False)
cfg.write(self.config_file)
self.config_file.close()
self.env_file = NamedTemporaryFile(mode='w', delete=False)
cfg = ConfigParser.ConfigParser()
cfg.add_section('some_section')
cfg.set('some_section', 'host', 'SOME_HOST_NAME')
cfg.set('some_section', 'port', 'SOME_PORT_NAME')
cfg.add_section('other_section')
cfg.set('other_section', 'foo', 'bar')
cfg.write(self.env_file)
self.env_file.close()
def tearDown(self):
self.config_file.unlink(self.config_file.name)
self.env_file.unlink(self.env_file.name)
def test_not_existing_section(self):
configuration = get_configuration('not-existing-section',
config_file=self.config_file.name)
self.assertIsNone(configuration)
def test_existing_config(self):
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'foo')
self.assertEquals(configuration['port'], '29192')
self.assertFalse('foo' in configuration)
def test_existing_env(self):
os.environ['SOME_HOST_NAME'] = 'bar'
os.environ['SOME_PORT_NAME'] = '6661'
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
self.assertTrue('host' in configuration)
self.assertTrue('port' in configuration)
self.assertEquals(configuration['host'], 'bar')
self.assertEquals(configuration['port'], '6661')
self.assertFalse('foo' in configuration)
os.environ.pop('SOME_HOST_NAME')
os.environ.pop('SOME_PORT_NAME')
def test_get_backend_class(self):
configuration = get_configuration(section_name='some_section',
config_file=self.config_file.name,
variables_file=self.env_file.name)
backend_class = get_backend_class(configuration)
self.assertTrue('Backend' in str(backend_class))
def store(self, file=None, content=None, ctype=None, **kwd):
"""save a file-like item"""
if content is None and not hasattr(file, 'read'):
raise TypeError('invalid file-like object')
data = content if content is not None else file.read()
size = len(data)
ext = guessImageType(data[:32])
if ext is None:
raise ValueError('invalid image file')
hashes = [md5(data).hexdigest()]
_exists_id = self.exists(hashed=hashes[0])
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print ('id {} or hash {} exists!!'.format(id, hashes[0]))
#raise DuplicateError('already exists')
return [True, id, filename]
ids = [_make_id(hashes[0])]
if 'id' in kwd and kwd['id'] and kwd['id'] not in ids:
ids += [kwd['id']]
from image import SimpImage, MIN_QUALITY
max_file_size = int(self.get_config('max_file_size'))
max_jpeg_quality = int(self.get_config('max_jpeg_quality'))
max_width = int(self.get_config('max_width'))
max_height = int(self.get_config('max_height'))
if size > max_file_size: max_jpeg_quality -= 1
if max_jpeg_quality < MIN_QUALITY: max_jpeg_quality = MIN_QUALITY
im = SimpImage(blob=data)
meta = im.meta
if meta['width'] > max_width or meta['height'] > max_height:
if self.get_config('auto_scale') and im.thumbnail(max_width, max_height):
if im.format == 'JPEG' and im.quality > max_jpeg_quality:
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
print im.meta
print 'new scaled size {}'.format(size)
hashes += [md5(data).hexdigest()]
else:
raise ValueError('file: {} dimension {}x{} is too big, max is {}x{}'.format(kwd['name'] if 'name' in kwd else '', meta['width'], meta['height'], max_width, max_height))
if im.format == 'JPEG':
if im.quality > max_jpeg_quality:
print 'quality {} is too high, hash {}'.format(im.quality, hashes[0])
from tempfile import NamedTemporaryFile
_tmp = NamedTemporaryFile('w+b',dir=self.get_config('temp_root'),delete=False)
_tmp.file.close()
save_file(_tmp.name, blob=data)
if jpegoptim(_tmp.name):
fp = open(_tmp.name)
data = fp.read()
size = len(data)
# print 'new optimized size {}'.format(size)
fp.close()
_tmp.unlink(_tmp.name)
del im
im = SimpImage(blob=data)
meta = im.meta
hashes += [md5(data).hexdigest()]
else:
raise EnvironmentError('jpeg qualty is too high, or need jpegoptim')
elif im.format == 'PNG' and self.get_config('force_jpeg'):
im.format = 'JPEG'
im.quality = max_jpeg_quality
data = im.get_blob()
size = len(data)
hashes += [md5(data).hexdigest()]
ext = 'jpg'
meta = im.meta
del im
if (size > max_file_size):
raise ValueError('file: {} size {} is too big, max is {}'.format(kwd['name'] if 'name' in kwd else '', size, max_file_size))
hashed = hashes[len(hashes)-1] #md5(data).hexdigest()
# print ('md5 hash: {}'.format(hashed))
# TODO: add for support (md5 + size) id
id = _make_id(hashed)
# print ('new filename: %r' % filename)
# TODO: fix for support s3 front browse
_exists_id = self.exists(id) or self.exists(hashed=hashed)
if _exists_id:
id = _exists_id
filename = _make_filename(id, ext)
print ('id {} or hash {} exists!!'.format(id, hashed))
#raise DuplicateError('already exists')
return [True, id, filename]
filename = _make_filename(id, ext)
# print ('id: {}'.format(id))
# if ctype is None or ctype == '':
from _util import guess_mimetype
ctype = guess_mimetype(filename)
# save to mongodb
spec = {'_id': id,'filename': filename, 'hash': hashes, 'mime': ctype, 'size': size, 'meta': meta, 'ids': ids}
if 'name' in kwd and isinstance(kwd['name'], (str, unicode)):
spec['name'] = kwd['name']
for k in ['created', 'app_id']:
if k in kwd and kwd[k]:
spec[k] = kwd[k]
if self._store_exists(id, filename=filename):
self._save_meta(id, spec)
return [True, id, filename]
rr = self._put(data, **spec)
if rr:
return [True, rr, filename]
def _merge(self, cc_dir):
md5_hashes = {}
if HASH_FILE.exists():
for line in [l.strip() for l in HASH_FILE.read_text().split("\n")]:
if line:
values = line.rsplit(":", maxsplit=1)
if len(values) == 2 and values[0] and values[1]:
md5_hashes[values[0]] = values[1]
try:
p = subprocess.run(
"git -C \"{cc_dir}\" status --porcelain -uall".format(
**locals()),
shell=True,
stdout=subprocess.PIPE,
check=True)
merge_files = []
status = str(p.stdout, "utf-8").strip()
for line in [s.strip() for s in status.split("\n")]:
if line:
merge_files.append(tuple(line.split()))
except subprocess.CalledProcessError as err:
raise nicfit.CommandError(str(err))
for st, file in merge_files + self.args.extra_merge:
dst = Path(file)
src = cc_dir / dst
hasher = md5()
try:
hasher.update(src.read_bytes())
except FileNotFoundError as notfound:
perr(notfound)
continue
md5sum = hasher.hexdigest()
merge_file = (self.args.ignore_md5s or file not in md5_hashes
or md5sum != md5_hashes[file])
pout("Comparing {} hash({}): {}".format(
file, md5sum,
Fg.blue("new") if merge_file else Fg.green("merged")))
md5_hashes[file] = md5sum
if merge_file:
tmp_dst = None
if not dst.exists():
tmp_dst = NamedTemporaryFile("w",
suffix=dst.suffix,
delete=False)
# Write the file to exist on disk for diff and merge
tmp_dst.close()
tmp_dst = Path(tmp_dst.name)
dst_file = str(dst if tmp_dst is None else tmp_dst)
diffs = subprocess.run("diff '{src}' '{dst_file}' >/dev/null"
.format(**locals()), shell=True)\
.returncode != 0
pout("Differences: {}".format(diffs))
if diffs:
merge_cmd = self.args.merge_cmd
if merge_cmd is None:
for cmd, opts in MERGE_TOOLS.items():
if shutil.which(cmd):
merge_cmd = " ".join([cmd, opts or ""])
break
if merge_cmd is not None:
subprocess.run(
"{merge_cmd} '{src}' '{dst_file}'".format(
**locals()),
shell=True,
check=True)
else:
perr("Merge disabled, no merge command found. Install "
"a merge tool such as: {tools}.\nOr use "
"--merge-cmd to specify your own.".format(
tools=", ".join(MERGE_TOOLS.keys())))
if tmp_dst and tmp_dst.stat().st_size == 0:
tmp_dst.unlink()
elif tmp_dst:
# Move tmp file into place and create parent dirs
if not dst.parent.exists():
dst.parent.mkdir(0o755, parents=True)
shutil.move(str(tmp_dst), str(dst))
with HASH_FILE.open("w") as hash_file:
for f in sorted(md5_hashes.keys()):
hash_file.write("{}:{}\n".format(f, md5_hashes[f]))
def handle(self, *args, **options):
if not GOOGLE_DOCS_ACCOUNT and GOOGLE_DOCS_PASSWORD and GOOGLE_DOCS_RESOURCE_ID:
raise CommandError(
'You must set both GOOGLE_DOCS_ACCOUNT, GOOGLE_DOCS_PASSWORD and GOOGLE_DOCS_RESOURCE_ID in your settings file.'
)
verbosity = int(options.get('verbosity', 1))
output_all = options.get('output_all')
dry_run = options.get('dry_run')
fields = ('email', 'first_name', 'last_name', 'phone', 'city', 'state',
'zipcode', 'is_a', 'broadcasters', 'date_created',
'share_info')
profile_list = NonUserProfile.objects.order_by('-date_created')
if not output_all:
profile_list = profile_list.filter(share_info=True)
if len(profile_list):
if verbosity > 1:
self.stdout.write('{0} signups to record.'.format(
len(profile_list)))
fp = NamedTemporaryFile(delete=False)
writer = csv.DictWriter(fp, fields)
writer.writeheader()
for signup in profile_list:
output = {
'email':
signup.email,
'first_name':
signup.first_name,
'last_name':
signup.last_name,
'phone':
signup.phone,
'city':
signup.city,
'state':
signup.state,
'zipcode':
signup.zipcode,
'is_a':
signup.is_a,
'date_created':
signup.date_created.strftime('%m/%d/%Y %H:%M:%S'),
'share_info':
signup.share_info
}
extra_fields_data = signup.extra_fields
for extra_field in SIGNUP_EXTRA_FIELDS:
input_val = None
if isinstance(extra_fields_data[extra_field], list):
input_val = ', '.join(extra_fields_data[extra_field])
else:
input_val = extra_fields_data[extra_field]
output[extra_field] = input_val
writer.writerow(output)
if dry_run:
self.stdout.write('Row created:\n{0}\n'.format('|'.join(
[str(output[f]) for f in fields])))
del (writer)
if not dry_run:
client = gdata.docs.client.DocsClient()
login_token = client.ClientLogin(GOOGLE_DOCS_ACCOUNT,
GOOGLE_DOCS_PASSWORD,
'politicaladsleuth')
fp.close()
media = gdata.data.MediaSource(file_path=fp.name,
content_type='text/csv')
try:
resource = client.get_resource_by_id(
GOOGLE_DOCS_RESOURCE_ID)
updated_resource = client.update_resource(
resource,
media=media,
update_metadata=False,
new_revision=True)
self.stdout.write('Data uploaded to "%s"\n'.format(
updated_resource.title.text))
except gdata.client.RequestError as e:
self.stderr.write(e.message + '\n')
self.stdout.write(
'****Upload may have succeeded despite an InvalidEntryException error****\n'
)
fp.close()
fp.unlink(fp.name)
else:
self.stdout.write('No signups for the given parameters\n')
