def do_ping(): d = {} d['metadata'] = metadata.get_metadata() d['info'] = metadata.get_container_info() d['timestamp'] = datetime.datetime.now().isoformat(' ') resp = requests.post(SERVER_URL, data=json.dumps(d), headers={'Content-Type': 'application/json'})
def iter_ms_layers(self, attr={}, meta={}, mra={}): def check(f, v): return f(v) if callable(f) else f == v for l in xrange(self.ms.numlayers): ms_layer = self.ms.getLayer(l) if not all(check(checker, getattr(ms_layer, k, None)) for k, checker in attr.iteritems()): continue if not all(check(checker, metadata.get_metadata(ms_layer, k, None)) for k, checker in meta.iteritems()): continue if not all(check(checker, metadata.get_mra_metadata(ms_layer, k, None)) for k, checker in mra.iteritems()): continue yield ms_layer
def cached(item): hash = item.hash() cacheFile = join(cacheFolder, hash) if not is_cached(item): from downloader import downloader downloader.add(item) return None if not item.got_metadata: metadata = get_metadata(cacheFile) item.artist = metadata.get("artistName", "") item.album = metadata.get("albumTitle", "") item.title = metadata.get("trackName", "") item.trackLength = metadata.get("totalTime", 0) item.trackNumber = metadata.get("trackNumber", "0") item.got_metadata = True item.save() return abspath(cacheFile)
def iter_ms_layers(self, attr={}, meta={}, mra={}): def check(f, v): return f(v) if callable(f) else f == v for l in xrange(self.ms.numlayers): ms_layer = self.ms.getLayer(l) if not all( check(checker, getattr(ms_layer, k, None)) for k, checker in attr.iteritems()): continue if not all( check(checker, metadata.get_metadata(ms_layer, k, None)) for k, checker in meta.iteritems()): continue if not all( check(checker, metadata.get_mra_metadata( ms_layer, k, None)) for k, checker in mra.iteritems()): continue yield ms_layer
def merge_video(params, temp_filenames, output_filename): mmg_command = 'mkvmerge -o %s ' % (output_filename) for index, temp_filename in enumerate(temp_filenames): temp_video_delay = get_metadata(params, temp_filename).get('delay') if index > 0: mmg_command += '--sync 0:%s + %s ' % (temp_video_delay, temp_filename) else: mmg_command += '%s ' % (temp_filename) if params.get('prompt'): print(mmg_command) else: start_external_execution(mmg_command) if os.path.isfile(output_filename): for filename in temp_filenames: print('Deleting: %s' % (os.path.abspath(filename))) os.remove(filename)
def main(): args = get_arguments() client = datastore.Client(project=args.project_id) query = client.query(kind=args.photo_table) entities = list(query.fetch()) results = [] print "Initial results:", len(entities) for entity in entities: m = metadata.get_metadata(entity, args.image_bucket, debug=False) if m is not None: results.append((entity.key.name, m)) print "Filtered results:", len(results) f = open(args.files, "wb") for result in results: f.write("%s/%s\n" % (args.directory, result[0])) f.close() pickle.dump(dict(results), open(args.output, "wb"))
def collection(self): while not self._event.is_set(): try: current_ts = time.monotonic() if self._meta_ts is None or ( current_ts - self._meta_ts ) >= self._config.get('host_metadata_interval'): metadata = get_metadata( get_hostname(), AGENT_VERSION, start_event=(self._meta_ts is None)) self._serializer.submit_metadata(metadata) self._meta_ts = current_ts self._collector.run_checks() self._serializer.serialize_and_push() except Exception: log.exception("Unexpected error in last collection run") time.sleep(self._config.get('min_collection_interval'))
import connections as conn from datetime import datetime from short_line import ShortLine from metadata import get_metadata pullStart = datetime.now() ## Get data about tables and columns mstables = get_metadata(conn.FromMSSQL) ## Create object, connect, and pull tables ## tables defined in connections.py for key in sorted(mstables.iterkeys()): sl = ShortLine(conn.FromMSSQL, conn.ToMySQL) sl.pull_data(sl, key, mstables[key]) pullStop = datetime.now() print "Time elapse of database sync was :", pullStop - pullStart
def process_feed(api, feed_conf, friends_cache, users_cache, urls_cache): data_dir = 'data/{}'.format(feed_conf['id']) if not os.path.exists(data_dir): os.makedirs(data_dir) db = Database(os.path.join(data_dir, 'db')) now = datetime.now().timestamp() last_seen_path = os.path.join(data_dir, 'last_seen') last_seen = util.try_load_json(last_seen_path) last_updated_path = os.path.join(data_dir, 'last_updated') last_updated = util.try_load_json(last_updated_path) last_update = max(last_updated.values()) if last_updated else 0 logger.info('Last updated: {}'.format(last_update)) compile_rss(db, last_update, data_dir, feed_conf) if 'friends' not in friends_cache: friends_cache['friends'] = [str(u_id) for u_id in tweepy.Cursor(api.friends_ids).items()] users = friends_cache['friends'] for l in feed_conf['lists']: if l not in friends_cache: user, slug = l.split('/') list_users = [str(u.id) for u in tweepy.Cursor(api.list_members, slug=slug, owner_screen_name=user).items()] friends_cache[l] = list_users users += friends_cache[l] users = set(users) users = {u: last_updated.get(u, -1) for u in users} users = sorted(list(users), key=lambda u: users[u]) logger.info('{} users'.format(len(users))) try: keywords = feed_conf.get('keywords', []) for i, user_id in enumerate(users): last = last_seen.get(user_id, None) logger.info('Fetching user {}, last fetched id: {}'.format(user_id, last)) try: if user_id in users_cache: tweets = users_cache[user_id] else: tweets = api.user_timeline(user_id=user_id, count=200, since_id=last, tweet_mode='extended') users_cache[user_id] = tweets except tweepy.TweepError: logger.error('Failed to fetch tweets for user {}, their tweets may be protected'.format(user_id)) continue for t in tweets: if not keywords or any(kw in t.full_text.lower() for kw in keywords): user = t.user.screen_name sub_statuses = [] urls = [url['expanded_url'] for url in t.entities['urls']] for attr in ['retweeted_status', 'quoted_status']: if hasattr(t, attr): sub_status = getattr(t, attr) urls += [url['expanded_url'] for url in sub_status.entities['urls']] sub_statuses.append({ 'id': sub_status.id_str, 'user': sub_status.user.screen_name, 'text': sub_status.full_text, }) for url in set(urls): if util.is_twitter_url(url): continue try: if url not in urls_cache: logger.info('Fetching metadata: {}'.format(url)) urls_cache[url] = get_metadata(url) meta = urls_cache[url] except Exception as e: logger.info('Error getting metadata for {}: {}'.format(url, e)) meta = {'url': url} # Sometimes the metadata canonical url will be a relative path, # if that's the case just stick with the url we have if meta['url'].startswith('http'): url = meta['url'] if util.is_twitter_url(url): continue logger.info('@{}: {}'.format(user, url)) db.inc(url, user) db.add_context(t.id_str, url, user, t.full_text, sub_statuses) last = last_seen.get(user_id, None) if last is None or t.id > last: last_seen[user_id] = t.id last_updated[user_id] = now with open(last_seen_path, 'w') as f: json.dump(last_seen, f) with open(last_updated_path, 'w') as f: json.dump(last_updated, f) if i % 100 == 0: compile_rss(db, last_update, data_dir, feed_conf) except tweepy.error.RateLimitError: logger.info('Rate limited') compile_rss(db, last_update, data_dir, feed_conf) logger.info('Done: {}'.format(feed_conf['id']))
def start(): """ Dummy start until we have a collector """ init_agent() hostname = get_hostname() logging.info("Starting the agent, hostname: %s", hostname) # init Forwarder logging.info("Starting the Forwarder") api_key = config.get('api_key') dd_url = config.get('dd_url') if not dd_url: logging.error('No Datadog URL configured - cannot continue') sys.exit(1) if not api_key: logging.error('No API key configured - cannot continue') sys.exit(1) forwarder = Forwarder(api_key, dd_url) forwarder.start() # aggregator aggregator = MetricsAggregator( hostname, interval=config.get('aggregator_interval'), expiry_seconds=(config.get('min_collection_interval') + config.get('aggregator_expiry_seconds')), recent_point_threshold=config.get('recent_point_threshold'), histogram_aggregates=config.get('histogram_aggregates'), histogram_percentiles=config.get('histogram_percentiles'), ) # serializer serializer = Serializer( aggregator, forwarder, ) # instantiate collector collector = Collector(config, aggregator) collector.load_check_classes() collector.instantiate_checks() def signal_handler(signal, frame): logging.info("SIGINT received: stopping the agent") logging.info("Stopping the forwarder") forwarder.stop() logging.info("See you !") sys.exit(0) signal.signal(signal.SIGINT, signal_handler) # update the metadata periodically? metadata = get_metadata(hostname) serializer.submit_metadata(metadata) while True: collector.run_checks() serializer.serialize_and_push() time.sleep(config.get('min_collection_interval'))
if imageformat == "JPEG": imageformat = "jpg" imgs = get_all_images("page.html") for img in imgs: try: if imageformat.lower() in img: download(img, 'images') except: print('Couldent download any images!!!') # webbrowser.open("page.html") webbrowser.open(res.url) print('\n\n********************************************') imagemeta = get_metadata(imagename) print('Before upload:') if (imagemeta): print(get_metadata(imagename)) else: print('No metadata found!!!') imagename = os.listdir('./images')[0] print('\n\nAfter upload:') if (imagemeta): print(get_metadata('./images/' + imagename)) else: print('No metadata found!!!') print('********************************************')
if params.get('ssa'): convert_to_ssa(params['in']) exit(0) if not params['in'].endswith('.avs'): params['avs'] = False print( 'Not an avscript. [Skipping custom commands processing from the given input]' ) params['source_file'] = params['in'] if params['config'] and params['config'].get('trims'): params['frame_rate'] = params['fr'] if params[ 'fr'] else get_frame_rate(params['in']) params['source_delay'] = get_metadata(params, params['in']).get('delay') times_list = get_trim_times(params, params['in'], params['frame_rate']) else: params['source_delay'] = 0 else: params['avs'] = True commands = get_custom_commands(params['in']) if commands.get('input'): params['source_file'] = os.path.join(os.path.dirname(params['in']), commands['input']) else: params['source_file'] = get_source(params['in'])
def get_file_gspath(user, path): sha1 = metadata.get_metadata(user, path)['sha1'] return gsfile.get_gs_path(sha1)
def file_metadata(user, path): ''' if file not exists, return None ''' return metadata.get_metadata(user, path)