def do_ping():
d = {}
d['metadata'] = metadata.get_metadata()
d['info'] = metadata.get_container_info()
d['timestamp'] = datetime.datetime.now().isoformat(' ')
resp = requests.post(SERVER_URL, data=json.dumps(d),
headers={'Content-Type': 'application/json'})
def iter_ms_layers(self, attr={}, meta={}, mra={}):
def check(f, v):
return f(v) if callable(f) else f == v
for l in xrange(self.ms.numlayers):
ms_layer = self.ms.getLayer(l)
if not all(check(checker, getattr(ms_layer, k, None)) for k, checker in attr.iteritems()):
continue
if not all(check(checker, metadata.get_metadata(ms_layer, k, None)) for k, checker in meta.iteritems()):
continue
if not all(check(checker, metadata.get_mra_metadata(ms_layer, k, None)) for k, checker in mra.iteritems()):
continue
yield ms_layer
def cached(item):
hash = item.hash()
cacheFile = join(cacheFolder, hash)
if not is_cached(item):
from downloader import downloader
downloader.add(item)
return None
if not item.got_metadata:
metadata = get_metadata(cacheFile)
item.artist = metadata.get("artistName", "")
item.album = metadata.get("albumTitle", "")
item.title = metadata.get("trackName", "")
item.trackLength = metadata.get("totalTime", 0)
item.trackNumber = metadata.get("trackNumber", "0")
item.got_metadata = True
item.save()
return abspath(cacheFile)
def iter_ms_layers(self, attr={}, meta={}, mra={}):
def check(f, v):
return f(v) if callable(f) else f == v
for l in xrange(self.ms.numlayers):
ms_layer = self.ms.getLayer(l)
if not all(
check(checker, getattr(ms_layer, k, None))
for k, checker in attr.iteritems()):
continue
if not all(
check(checker, metadata.get_metadata(ms_layer, k, None))
for k, checker in meta.iteritems()):
continue
if not all(
check(checker, metadata.get_mra_metadata(
ms_layer, k, None)) for k, checker in mra.iteritems()):
continue
yield ms_layer
def merge_video(params, temp_filenames, output_filename):
mmg_command = 'mkvmerge -o %s ' % (output_filename)
for index, temp_filename in enumerate(temp_filenames):
temp_video_delay = get_metadata(params, temp_filename).get('delay')
if index > 0:
mmg_command += '--sync 0:%s + %s ' % (temp_video_delay,
temp_filename)
else:
mmg_command += '%s ' % (temp_filename)
if params.get('prompt'):
print(mmg_command)
else:
start_external_execution(mmg_command)
if os.path.isfile(output_filename):
for filename in temp_filenames:
print('Deleting: %s' % (os.path.abspath(filename)))
os.remove(filename)
def main():
args = get_arguments()
client = datastore.Client(project=args.project_id)
query = client.query(kind=args.photo_table)
entities = list(query.fetch())
results = []
print "Initial results:", len(entities)
for entity in entities:
m = metadata.get_metadata(entity, args.image_bucket, debug=False)
if m is not None:
results.append((entity.key.name, m))
print "Filtered results:", len(results)
f = open(args.files, "wb")
for result in results:
f.write("%s/%s\n" % (args.directory, result[0]))
f.close()
pickle.dump(dict(results), open(args.output, "wb"))
def collection(self):
while not self._event.is_set():
try:
current_ts = time.monotonic()
if self._meta_ts is None or (
current_ts - self._meta_ts
) >= self._config.get('host_metadata_interval'):
metadata = get_metadata(
get_hostname(),
AGENT_VERSION,
start_event=(self._meta_ts is None))
self._serializer.submit_metadata(metadata)
self._meta_ts = current_ts
self._collector.run_checks()
self._serializer.serialize_and_push()
except Exception:
log.exception("Unexpected error in last collection run")
time.sleep(self._config.get('min_collection_interval'))
import connections as conn
from datetime import datetime
from short_line import ShortLine
from metadata import get_metadata
pullStart = datetime.now()
## Get data about tables and columns
mstables = get_metadata(conn.FromMSSQL)
## Create object, connect, and pull tables
## tables defined in connections.py
for key in sorted(mstables.iterkeys()):
sl = ShortLine(conn.FromMSSQL, conn.ToMySQL)
sl.pull_data(sl, key, mstables[key])
pullStop = datetime.now()
print "Time elapse of database sync was :", pullStop - pullStart
def process_feed(api, feed_conf, friends_cache, users_cache, urls_cache):
data_dir = 'data/{}'.format(feed_conf['id'])
if not os.path.exists(data_dir):
os.makedirs(data_dir)
db = Database(os.path.join(data_dir, 'db'))
now = datetime.now().timestamp()
last_seen_path = os.path.join(data_dir, 'last_seen')
last_seen = util.try_load_json(last_seen_path)
last_updated_path = os.path.join(data_dir, 'last_updated')
last_updated = util.try_load_json(last_updated_path)
last_update = max(last_updated.values()) if last_updated else 0
logger.info('Last updated: {}'.format(last_update))
compile_rss(db, last_update, data_dir, feed_conf)
if 'friends' not in friends_cache:
friends_cache['friends'] = [str(u_id) for u_id in tweepy.Cursor(api.friends_ids).items()]
users = friends_cache['friends']
for l in feed_conf['lists']:
if l not in friends_cache:
user, slug = l.split('/')
list_users = [str(u.id) for u in tweepy.Cursor(api.list_members, slug=slug, owner_screen_name=user).items()]
friends_cache[l] = list_users
users += friends_cache[l]
users = set(users)
users = {u: last_updated.get(u, -1) for u in users}
users = sorted(list(users), key=lambda u: users[u])
logger.info('{} users'.format(len(users)))
try:
keywords = feed_conf.get('keywords', [])
for i, user_id in enumerate(users):
last = last_seen.get(user_id, None)
logger.info('Fetching user {}, last fetched id: {}'.format(user_id, last))
try:
if user_id in users_cache:
tweets = users_cache[user_id]
else:
tweets = api.user_timeline(user_id=user_id, count=200, since_id=last, tweet_mode='extended')
users_cache[user_id] = tweets
except tweepy.TweepError:
logger.error('Failed to fetch tweets for user {}, their tweets may be protected'.format(user_id))
continue
for t in tweets:
if not keywords or any(kw in t.full_text.lower() for kw in keywords):
user = t.user.screen_name
sub_statuses = []
urls = [url['expanded_url'] for url in t.entities['urls']]
for attr in ['retweeted_status', 'quoted_status']:
if hasattr(t, attr):
sub_status = getattr(t, attr)
urls += [url['expanded_url'] for url in sub_status.entities['urls']]
sub_statuses.append({
'id': sub_status.id_str,
'user': sub_status.user.screen_name,
'text': sub_status.full_text,
})
for url in set(urls):
if util.is_twitter_url(url): continue
try:
if url not in urls_cache:
logger.info('Fetching metadata: {}'.format(url))
urls_cache[url] = get_metadata(url)
meta = urls_cache[url]
except Exception as e:
logger.info('Error getting metadata for {}: {}'.format(url, e))
meta = {'url': url}
# Sometimes the metadata canonical url will be a relative path,
# if that's the case just stick with the url we have
if meta['url'].startswith('http'):
url = meta['url']
if util.is_twitter_url(url): continue
logger.info('@{}: {}'.format(user, url))
db.inc(url, user)
db.add_context(t.id_str, url, user, t.full_text, sub_statuses)
last = last_seen.get(user_id, None)
if last is None or t.id > last: last_seen[user_id] = t.id
last_updated[user_id] = now
with open(last_seen_path, 'w') as f:
json.dump(last_seen, f)
with open(last_updated_path, 'w') as f:
json.dump(last_updated, f)
if i % 100 == 0:
compile_rss(db, last_update, data_dir, feed_conf)
except tweepy.error.RateLimitError:
logger.info('Rate limited')
compile_rss(db, last_update, data_dir, feed_conf)
logger.info('Done: {}'.format(feed_conf['id']))
def start():
"""
Dummy start until we have a collector
"""
init_agent()
hostname = get_hostname()
logging.info("Starting the agent, hostname: %s", hostname)
# init Forwarder
logging.info("Starting the Forwarder")
api_key = config.get('api_key')
dd_url = config.get('dd_url')
if not dd_url:
logging.error('No Datadog URL configured - cannot continue')
sys.exit(1)
if not api_key:
logging.error('No API key configured - cannot continue')
sys.exit(1)
forwarder = Forwarder(api_key, dd_url)
forwarder.start()
# aggregator
aggregator = MetricsAggregator(
hostname,
interval=config.get('aggregator_interval'),
expiry_seconds=(config.get('min_collection_interval') +
config.get('aggregator_expiry_seconds')),
recent_point_threshold=config.get('recent_point_threshold'),
histogram_aggregates=config.get('histogram_aggregates'),
histogram_percentiles=config.get('histogram_percentiles'),
)
# serializer
serializer = Serializer(
aggregator,
forwarder,
)
# instantiate collector
collector = Collector(config, aggregator)
collector.load_check_classes()
collector.instantiate_checks()
def signal_handler(signal, frame):
logging.info("SIGINT received: stopping the agent")
logging.info("Stopping the forwarder")
forwarder.stop()
logging.info("See you !")
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
# update the metadata periodically?
metadata = get_metadata(hostname)
serializer.submit_metadata(metadata)
while True:
collector.run_checks()
serializer.serialize_and_push()
time.sleep(config.get('min_collection_interval'))
if imageformat == "JPEG":
imageformat = "jpg"
imgs = get_all_images("page.html")
for img in imgs:
try:
if imageformat.lower() in img:
download(img, 'images')
except:
print('Couldent download any images!!!')
# webbrowser.open("page.html")
webbrowser.open(res.url)
print('\n\n********************************************')
imagemeta = get_metadata(imagename)
print('Before upload:')
if (imagemeta):
print(get_metadata(imagename))
else:
print('No metadata found!!!')
imagename = os.listdir('./images')[0]
print('\n\nAfter upload:')
if (imagemeta):
print(get_metadata('./images/' + imagename))
else:
print('No metadata found!!!')
print('********************************************')
if params.get('ssa'):
convert_to_ssa(params['in'])
exit(0)
if not params['in'].endswith('.avs'):
params['avs'] = False
print(
'Not an avscript. [Skipping custom commands processing from the given input]'
)
params['source_file'] = params['in']
if params['config'] and params['config'].get('trims'):
params['frame_rate'] = params['fr'] if params[
'fr'] else get_frame_rate(params['in'])
params['source_delay'] = get_metadata(params,
params['in']).get('delay')
times_list = get_trim_times(params, params['in'],
params['frame_rate'])
else:
params['source_delay'] = 0
else:
params['avs'] = True
commands = get_custom_commands(params['in'])
if commands.get('input'):
params['source_file'] = os.path.join(os.path.dirname(params['in']),
commands['input'])
else:
params['source_file'] = get_source(params['in'])
def get_file_gspath(user, path):
sha1 = metadata.get_metadata(user, path)['sha1']
return gsfile.get_gs_path(sha1)
def file_metadata(user, path):
''' if file not exists, return None '''
return metadata.get_metadata(user, path)
