def setUp(self):
self.config = test.config()
self.config['stage'] = '/data/home/{0}/.test_apm'.format(self.config['username'])
self.stage = self.config['stage']
if not os.path.exists(self.config['stage']):
os.makedirs(self.config['stage'])
def setUp(self):
self.stage = None
self.source = None
self.config = test.config()
self.message = "Please enter a directory: "
self.error = "A directory is required. Please try again."
self.default = {
'stage': '/data/home/{0}/.test_apm'.format(self.config['username']),
'source': '/data/archive'
}
self.alt = {
'stage': {
'not_exists': '/data/stage',
'not_writable': '/data/archive',
'new': '/data/home/{0}/.test_apm/stage'.format(self.config['username'])
},
'source': {
'not_exists': '/data/home/{0}/.test/source'.format(self.config['username']),
'empty': '/data/home/{0}/.test_apm/empty_source'.format(self.config['username'])
}
}
dirs = [self.default['stage'], self.alt['source']['empty']]
for i in dirs:
if not os.path.exists(i):
os.makedirs(i)
def setUp(self):
today = date.fromtimestamp(time.time())
self.config = test.config()
self.begin = self.config['begin']
self.end = self.config['end']
self.today = int(today.strftime("%Y%m%d"))
self.earlier = int((today - timedelta(days=3)).strftime("%Y%m%d"))
self.later = int((today + timedelta(days=3)).strftime("%Y%m%d"))
self.evenLater = int((today + timedelta(days=6)).strftime("%Y%m%d"))
self.message = "Please enter a date (YYYYMMDD): "
def setUp(self):
self.config = test.config()
def main():
if not len(sys.argv) > 1:
sys.argv.append("-h")
if '-v' in sys.argv:
print(apm.__version__)
return
# Retrieve arguments from user
config = parse_args()
command = config['command'].lower()
# Check to see if this is a test
if command == 'test':
test_config = test.config()
sys.argv = [sys.argv[0]]
jprint(config, sort_keys=True, indent=4)
unittest.main(buffer=True)
# unittest.main()
return
# Not a test
if command == 'info' or command == 'vapinfo':
vap = VapMgr({})
vap.vap_info()
return
if command == 'auto':
temp = validate_config(config, command)
config = temp if temp else config
else:
# Validate user arguments
temp = validate_config(config, command)
config = temp if temp else config
if command == 'check':
jprint(config, sort_keys=True, indent=4)
return
# Save the config to file
s = time.time()
f = Files(config)
f.save_config()
f.load_filenames()
files = f.files
# Check to see if any files are not currently being tracked
# Or if any tracked files have been deleted
print("Checking status of tracked files...", end="")
sys.stdout.flush()
json_file = '{0}/{1}/{1}.json'.format(config['stage'], config['job'])
if os.path.exists(json_file) and config['ingest']:
fp = open(json_file, 'r')
files = json.loads(fp.read())
fp.close()
cwd = os.getcwd()
os.chdir('{}/{}/collection'.format(config['stage'], config['job']))
keys = files.keys()
sites = set(os.listdir('.'))
for site in keys:
if site not in sites:
files.pop(site)
continue
os.chdir(site)
instruments = set(os.listdir('.'))
ins_keys = files[site].keys()
for ins in ins_keys:
if ins not in instruments:
files[site].pop(ins)
continue
os.chdir(ins)
filelist = set(os.listdir('.'))
for i in filelist:
if i not in files[site][ins] and not (os.path.isdir(i) and i == "other_files"):
exit("\nThe file {0}/{1}/{2} is currently untracked.\nPlease edit {3}.json to start tracking this file.\n".format(site, ins, i, config['job']))
for i in files[site][ins]:
if i not in filelist:
files[site][ins][i]["deleted"] = True
os.chdir('..')
os.chdir('..')
os.chdir(cwd)
print("Done") # Done checking status of tracked files
sys.stdout.flush()
# Run the appropriate command
if command == 'auto':
print('Attempting to stage files for datastreams: {}'.format(config['datastream']))
skip = False
if not config['duplicates']:
s = Stage(config, files)
config, files = s.run()
if config['exit']:
exit()
if config['duplicates']:
skip = True
if not skip and not config['vap']:
r = Rename(config, files)
config, files = r.run()
if config['exit']:
exit()
exit()
elif command == 'stage':
print("*"*50,"\n", json.dumps(config, indent=2), "*"*50, "\n")
skip = False
if not config['duplicates']:
s = Stage(config, files)
config, files = s.run()
if config['exit']:
exit()
if config['duplicates']:
skip = True
if not skip and not config['vap']:
r = Rename(config, files)
config, files = r.run()
if config['exit']:
exit()
elif command == 'rename':
# If rename is called explicitly, force rename even if config is set to false
switch = True if config['rename'] == False else False
if switch:
config['rename'] = True
if not config['vap']:
r = Rename(config, files)
config, files = r.run()
if config['exit']:
exit()
if switch:
config['rename'] = False
elif command == 'process':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
p = Process(config, files)
config, files = p.run()
if config['exit']:
exit()
elif command == 'review':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
r = Review(config, files)
config, files = r.run()
if config['exit']:
exit()
elif command == 'remove':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
r = Remove(config, files)
config, files = r.run()
if config['exit']:
exit()
elif command == 'archive':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
a = Archive(config, files)
config, files = a.run()
if config['exit']:
exit()
elif command == 'cleanup':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
c = Cleanup(config, files)
config, files = c.run()
if config['exit']:
exit()
elif command == 'prep':
r = Rename(config, files)
has_coll = r.check_for_collisions()
files = r.files
if has_coll:
config = r.config
files = r.files
else:
d = Demo(config, files)
config, files = d.run()
if config['exit']:
exit()
elif command == "notification":
# Alka's module goes here
print('Yay notify the user shit has changed.')
else:
sys.argv.append("-h")
config = parse_args()
f.config = config
f.files = files
f.save_config()
f.save_filenames()